shardItems) {
this.shardItems = shardItems;
}
public String getParams() {
return params;
}
public void setParams(String params) {
this.params = params;
}
public String getTaskId() {
return taskId;
}
public void setTaskId(String taskId) {
this.taskId = taskId;
}
public String getApplicationName() {
return applicationName;
}
public void setApplicationName(String applicationName) {
this.applicationName = applicationName;
}
public String getGroupName() {
return groupName;
}
public void setGroupName(String groupName) {
this.groupName = groupName;
}
public String getTaskName() {
return taskName;
}
public void setTaskName(String taskName) {
this.taskName = taskName;
}
public int getConnectorPort() {
return connectorPort;
}
public void setConnectorPort(int connectorPort) {
this.connectorPort = connectorPort;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public String getClassName() {
return className;
}
public void setClassName(String className) {
this.className = className;
}
public String getMethodName() {
return methodName;
}
public void setMethodName(String methodName) {
this.methodName = methodName;
}
/**
* package request command
*
* @return command
*/
public Command convert2Command(){
Command command = new Command();
command.setType(CommandType.EXECUTE_TASK_REQUEST);
byte[] body = FastJsonSerializer.serialize(this);
command.setBody(body);
return command;
}
}
\ No newline at end of file
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java
new file mode 100644
index 0000000000..7e35fa6e75
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java
@@ -0,0 +1 @@
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
import java.util.concurrent.atomic.AtomicLong;
/**
* execute task response command
*/
public class ExecuteTaskResponseCommand implements Serializable {
/**
* task id
*/
private String taskId;
/**
* attempt id
*/
private String attemptId;
/**
* return result
*/
private Object result;
/**
* received time
*/
private long receivedTime;
/**
* execute count
*/
private int executeCount;
/**
* execute time
*/
private long executeTime;
public String getAttemptId() {
return attemptId;
}
public void setAttemptId(String attemptId) {
this.attemptId = attemptId;
}
public String getTaskId() {
return taskId;
}
public void setTaskId(String taskId) {
this.taskId = taskId;
}
public Object getResult() {
return result;
}
public void setResult(Object result) {
this.result = result;
}
public long getReceivedTime() {
return receivedTime;
}
public void setReceivedTime(long receivedTime) {
this.receivedTime = receivedTime;
}
public int getExecuteCount() {
return executeCount;
}
public void setExecuteCount(int executeCount) {
this.executeCount = executeCount;
}
public long getExecuteTime() {
return executeTime;
}
public void setExecuteTime(long executeTime) {
this.executeTime = executeTime;
}
public Command convert2Command(long opaque){
Command command = new Command();
command.setType(CommandType.EXECUTE_TASK_RESPONSE);
byte[] body = FastJsonSerializer.serialize(this);
command.setBody(body);
return command;
}
}
\ No newline at end of file
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java
new file mode 100644
index 0000000000..c50413e98a
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.command;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * ping machine
+ */
+public class Ping implements Serializable {
+
+ /**
+ * ping body
+ */
+ protected static ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER;
+
+ /**
+ * request command body
+ */
+ private static byte[] EMPTY_BODY_ARRAY = new byte[0];
+
+ private static final ByteBuf PING_BUF;
+
+ static {
+ ByteBuf ping = Unpooled.buffer();
+ ping.writeByte(Command.MAGIC);
+ ping.writeByte(CommandType.PING.ordinal());
+ ping.writeLong(0);
+ ping.writeInt(0);
+ ping.writeBytes(EMPTY_BODY);
+ PING_BUF = Unpooled.unreleasableBuffer(ping).asReadOnly();
+ }
+
+ /**
+ * ping content
+ * @return result
+ */
+ public static ByteBuf pingContent(){
+ return PING_BUF.duplicate();
+ }
+
+ /**
+ * create ping command
+ *
+ * @return command
+ */
+ public static Command create(){
+ Command command = new Command();
+ command.setType(CommandType.PING);
+ command.setBody(EMPTY_BODY_ARRAY);
+ return command;
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java
new file mode 100644
index 0000000000..e52cef6d92
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.command;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+
+import java.io.Serializable;
+
+/**
+ * Pong return after ping
+ */
+public class Pong implements Serializable {
+
+ /**
+ * pong body
+ */
+ protected static ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER;
+
+ /**
+ * pong command body
+ */
+ private static byte[] EMPTY_BODY_ARRAY = new byte[0];
+
+ /**
+ * ping byte buffer
+ */
+ private static final ByteBuf PONG_BUF;
+
+ static {
+ ByteBuf ping = Unpooled.buffer();
+ ping.writeByte(Command.MAGIC);
+ ping.writeByte(CommandType.PONG.ordinal());
+ ping.writeLong(0);
+ ping.writeInt(0);
+ ping.writeBytes(EMPTY_BODY);
+ PONG_BUF = Unpooled.unreleasableBuffer(ping).asReadOnly();
+ }
+
+ /**
+ * ping content
+ * @return result
+ */
+ public static ByteBuf pingContent(){
+ return PONG_BUF.duplicate();
+ }
+
+ /**
+ * package pong command
+ *
+ * @param opaque request unique identification
+ * @return command
+ */
+ public static Command create(long opaque){
+ Command command = new Command(opaque);
+ command.setType(CommandType.PONG);
+ command.setBody(EMPTY_BODY_ARRAY);
+ return command;
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java
new file mode 100644
index 0000000000..4cc32ed42a
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.command.log;
+
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
+
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * get log bytes request command
+ */
+public class GetLogBytesRequestCommand implements Serializable {
+
+ /**
+ * log path
+ */
+ private String path;
+
+ public GetLogBytesRequestCommand() {
+ }
+
+ public GetLogBytesRequestCommand(String path) {
+ this.path = path;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ /**
+ * package request command
+ *
+ * @return command
+ */
+ public Command convert2Command(){
+ Command command = new Command();
+ command.setType(CommandType.GET_LOG_BYTES_REQUEST);
+ byte[] body = FastJsonSerializer.serialize(this);
+ command.setBody(body);
+ return command;
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java
new file mode 100644
index 0000000000..deaf9b8d85
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.command.log;
+
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
+
+import java.io.Serializable;
+
+/**
+ * get log bytes response command
+ */
+public class GetLogBytesResponseCommand implements Serializable {
+
+ /**
+ * log byte data
+ */
+ private byte[] data;
+
+ public GetLogBytesResponseCommand() {
+ }
+
+ public GetLogBytesResponseCommand(byte[] data) {
+ this.data = data;
+ }
+
+ public byte[] getData() {
+ return data;
+ }
+
+ public void setData(byte[] data) {
+ this.data = data;
+ }
+
+ /**
+ * package response command
+ *
+ * @param opaque request unique identification
+ * @return command
+ */
+ public Command convert2Command(long opaque){
+ Command command = new Command(opaque);
+ command.setType(CommandType.GET_LOG_BYTES_RESPONSE);
+ byte[] body = FastJsonSerializer.serialize(this);
+ command.setBody(body);
+ return command;
+ }
+
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java
new file mode 100644
index 0000000000..621d35a804
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.command.log;
+
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
+
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * roll view log request command
+ */
+public class RollViewLogRequestCommand implements Serializable {
+
+ /**
+ * log path
+ */
+ private String path;
+
+ /**
+ * skip line number
+ */
+ private int skipLineNum;
+
+ /**
+ * query line number
+ */
+ private int limit;
+
+ public RollViewLogRequestCommand() {
+ }
+
+ public RollViewLogRequestCommand(String path, int skipLineNum, int limit) {
+ this.path = path;
+ this.skipLineNum = skipLineNum;
+ this.limit = limit;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public int getSkipLineNum() {
+ return skipLineNum;
+ }
+
+ public void setSkipLineNum(int skipLineNum) {
+ this.skipLineNum = skipLineNum;
+ }
+
+ public int getLimit() {
+ return limit;
+ }
+
+ public void setLimit(int limit) {
+ this.limit = limit;
+ }
+
+ /**
+ * package request command
+ *
+ * @return command
+ */
+ public Command convert2Command(){
+ Command command = new Command();
+ command.setType(CommandType.ROLL_VIEW_LOG_REQUEST);
+ byte[] body = FastJsonSerializer.serialize(this);
+ command.setBody(body);
+ return command;
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java
new file mode 100644
index 0000000000..591d787200
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.command.log;
+
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
+
+import java.io.Serializable;
+
+/**
+ * roll view log response command
+ */
+public class RollViewLogResponseCommand implements Serializable {
+
+ /**
+ * response data
+ */
+ private String msg;
+
+ public RollViewLogResponseCommand() {
+ }
+
+ public RollViewLogResponseCommand(String msg) {
+ this.msg = msg;
+ }
+
+ public String getMsg() {
+ return msg;
+ }
+
+ public void setMsg(String msg) {
+ this.msg = msg;
+ }
+
+ /**
+ * package response command
+ *
+ * @param opaque request unique identification
+ * @return command
+ */
+ public Command convert2Command(long opaque){
+ Command command = new Command(opaque);
+ command.setType(CommandType.ROLL_VIEW_LOG_RESPONSE);
+ byte[] body = FastJsonSerializer.serialize(this);
+ command.setBody(body);
+ return command;
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java
new file mode 100644
index 0000000000..8835348ee3
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.command.log;
+
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
+
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * view log request command
+ */
+public class ViewLogRequestCommand implements Serializable {
+
+ /**
+ * log path
+ */
+ private String path;
+
+ public ViewLogRequestCommand() {
+ }
+
+ public ViewLogRequestCommand(String path) {
+ this.path = path;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ /**
+ * package request command
+ *
+ * @return command
+ */
+ public Command convert2Command(){
+ Command command = new Command();
+ command.setType(CommandType.VIEW_WHOLE_LOG_REQUEST);
+ byte[] body = FastJsonSerializer.serialize(this);
+ command.setBody(body);
+ return command;
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java
new file mode 100644
index 0000000000..dffadade26
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.command.log;
+
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
+
+import java.io.Serializable;
+
+/**
+ * view log response command
+ */
+public class ViewLogResponseCommand implements Serializable {
+
+ /**
+ * response data
+ */
+ private String msg;
+
+ public ViewLogResponseCommand() {
+ }
+
+ public ViewLogResponseCommand(String msg) {
+ this.msg = msg;
+ }
+
+ public String getMsg() {
+ return msg;
+ }
+
+ public void setMsg(String msg) {
+ this.msg = msg;
+ }
+
+ /**
+ * package response command
+ *
+ * @param opaque request unique identification
+ * @return command
+ */
+ public Command convert2Command(long opaque){
+ Command command = new Command(opaque);
+ command.setType(CommandType.VIEW_WHOLE_LOG_RESPONSE);
+ byte[] body = FastJsonSerializer.serialize(this);
+ command.setBody(body);
+ return command;
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java
new file mode 100644
index 0000000000..831e05f7e7
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.remote.config;
+
+import org.apache.dolphinscheduler.remote.utils.Constants;
+
+/**
+ * netty client config
+ */
+public class NettyClientConfig {
+
+ /**
+ * worker threads，default get machine cpus
+ */
+ private int workerThreads = Constants.CPUS;
+
+ /**
+ * whether tpc delay
+ */
+ private boolean tcpNoDelay = true;
+
+ /**
+ * whether keep alive
+ */
+ private boolean soKeepalive = true;
+
+ /**
+ * send buffer size
+ */
+ private int sendBufferSize = 65535;
+
+ /**
+ * receive buffer size
+ */
+ private int receiveBufferSize = 65535;
+
+ public int getWorkerThreads() {
+ return workerThreads;
+ }
+
+ public void setWorkerThreads(int workerThreads) {
+ this.workerThreads = workerThreads;
+ }
+
+ public boolean isTcpNoDelay() {
+ return tcpNoDelay;
+ }
+
+ public void setTcpNoDelay(boolean tcpNoDelay) {
+ this.tcpNoDelay = tcpNoDelay;
+ }
+
+ public boolean isSoKeepalive() {
+ return soKeepalive;
+ }
+
+ public void setSoKeepalive(boolean soKeepalive) {
+ this.soKeepalive = soKeepalive;
+ }
+
+ public int getSendBufferSize() {
+ return sendBufferSize;
+ }
+
+ public void setSendBufferSize(int sendBufferSize) {
+ this.sendBufferSize = sendBufferSize;
+ }
+
+ public int getReceiveBufferSize() {
+ return receiveBufferSize;
+ }
+
+ public void setReceiveBufferSize(int receiveBufferSize) {
+ this.receiveBufferSize = receiveBufferSize;
+ }
+
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java
new file mode 100644
index 0000000000..4ec8a0f7a7
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.remote.config;
+
+import org.apache.dolphinscheduler.remote.utils.Constants;
+
+/**
+ * netty server config
+ */
+public class NettyServerConfig {
+
+ /**
+ * init the server connectable queue
+ */
+ private int soBacklog = 1024;
+
+ /**
+ * whether tpc delay
+ */
+ private boolean tcpNoDelay = true;
+
+ /**
+ * whether keep alive
+ */
+ private boolean soKeepalive = true;
+
+ /**
+ * send buffer size
+ */
+ private int sendBufferSize = 65535;
+
+ /**
+ * receive buffer size
+ */
+ private int receiveBufferSize = 65535;
+
+ /**
+ * worker threads，default get machine cpus
+ */
+ private int workerThread = Constants.CPUS;
+
+ /**
+ * listen port
+ */
+ private int listenPort = 12346;
+
+ public int getListenPort() {
+ return listenPort;
+ }
+
+ public void setListenPort(int listenPort) {
+ this.listenPort = listenPort;
+ }
+
+ public int getSoBacklog() {
+ return soBacklog;
+ }
+
+ public void setSoBacklog(int soBacklog) {
+ this.soBacklog = soBacklog;
+ }
+
+ public boolean isTcpNoDelay() {
+ return tcpNoDelay;
+ }
+
+ public void setTcpNoDelay(boolean tcpNoDelay) {
+ this.tcpNoDelay = tcpNoDelay;
+ }
+
+ public boolean isSoKeepalive() {
+ return soKeepalive;
+ }
+
+ public void setSoKeepalive(boolean soKeepalive) {
+ this.soKeepalive = soKeepalive;
+ }
+
+ public int getSendBufferSize() {
+ return sendBufferSize;
+ }
+
+ public void setSendBufferSize(int sendBufferSize) {
+ this.sendBufferSize = sendBufferSize;
+ }
+
+ public int getReceiveBufferSize() {
+ return receiveBufferSize;
+ }
+
+ public void setReceiveBufferSize(int receiveBufferSize) {
+ this.receiveBufferSize = receiveBufferSize;
+ }
+
+ public int getWorkerThread() {
+ return workerThread;
+ }
+
+ public void setWorkerThread(int workerThread) {
+ this.workerThread = workerThread;
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java
new file mode 100644
index 0000000000..29d48db8f8
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.exceptions;
+
+/**
+ * remote exception
+ */
+public class RemotingException extends Exception {
+
+ public RemotingException() {
+ super();
+ }
+
+ /** Constructs a new runtime exception with the specified detail message.
+ * The cause is not initialized, and may subsequently be initialized by a
+ * call to {@link #initCause}.
+ *
+ * @param message the detail message. The detail message is saved for
+ * later retrieval by the {@link #getMessage()} method.
+ */
+ public RemotingException(String message) {
+ super(message);
+ }
+
+ /**
+ * Constructs a new runtime exception with the specified detail message and
+ * cause. Note that the detail message associated with
+ * {@code cause} is not automatically incorporated in
+ * this runtime exception's detail message.
+ *
+ * @param message the detail message (which is saved for later retrieval
+ * by the {@link #getMessage()} method).
+ * @param cause the cause (which is saved for later retrieval by the
+ * {@link #getCause()} method). (A null value is
+ * permitted, and indicates that the cause is nonexistent or
+ * unknown.)
+ * @since 1.4
+ */
+ public RemotingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /** Constructs a new runtime exception with the specified cause and a
+ * detail message of (cause==null ? null : cause.toString())
+ * (which typically contains the class and detail message of
+ * cause). This constructor is useful for runtime exceptions
+ * that are little more than wrappers for other throwables.
+ *
+ * @param cause the cause (which is saved for later retrieval by the
+ * {@link #getCause()} method). (A null value is
+ * permitted, and indicates that the cause is nonexistent or
+ * unknown.)
+ * @since 1.4
+ */
+ public RemotingException(Throwable cause) {
+ super(cause);
+ }
+
+ /**
+ * Constructs a new runtime exception with the specified detail
+ * message, cause, suppression enabled or disabled, and writable
+ * stack trace enabled or disabled.
+ *
+ * @param message the detail message.
+ * @param cause the cause. (A {@code null} value is permitted,
+ * and indicates that the cause is nonexistent or unknown.)
+ * @param enableSuppression whether or not suppression is enabled
+ * or disabled
+ * @param writableStackTrace whether or not the stack trace should
+ * be writable
+ *
+ * @since 1.7
+ */
+ protected RemotingException(String message, Throwable cause,
+ boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java
new file mode 100644
index 0000000000..3d91ba57f6
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.exceptions;
+
+
+/**
+ * timeout exception
+ */
+public class RemotingTimeoutException extends RemotingException{
+
+ public RemotingTimeoutException(String message) {
+ super(message);
+ }
+
+
+ public RemotingTimeoutException(String address, long timeoutMillis) {
+ this(address, timeoutMillis, null);
+ }
+
+ public RemotingTimeoutException(String address, long timeoutMillis, Throwable cause) {
+ super(String.format("wait response on the channel %s timeout %s", address, timeoutMillis), cause);
+ }
+}
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/TestZkServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java
similarity index 65%
rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/TestZkServer.java
rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java
index d1a0526309..82cc3f4dbf 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/TestZkServer.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java
@@ -14,30 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.zk;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+package org.apache.dolphinscheduler.remote.exceptions;
/**
- * demo for using zkServer
+ * too much request exception
*/
-public class TestZkServer {
-
- @Before
- public void before(){
- ZKServer.start();
- }
-
- @Test
- public void test(){
- Assert.assertTrue(ZKServer.isStarted());
- }
+public class RemotingTooMuchRequestException extends RemotingException{
- @After
- public void after(){
- ZKServer.stop();
+ public RemotingTooMuchRequestException(String message) {
+ super(message);
}
}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/InvokeCallback.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/InvokeCallback.java
new file mode 100644
index 0000000000..84cdae867b
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/InvokeCallback.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.remote.future;
+
+/**
+ * invoke callback
+ */
+public interface InvokeCallback {
+
+ /**
+ * operation
+ *
+ * @param responseFuture responseFuture
+ */
+ void operationComplete(final ResponseFuture responseFuture);
+
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java
new file mode 100644
index 0000000000..95a04b1f1a
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.remote.future;
+
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * release semaphore
+ */
+public class ReleaseSemaphore {
+
+ private final Semaphore semaphore;
+
+ private final AtomicBoolean released;
+
+ public ReleaseSemaphore(Semaphore semaphore){
+ this.semaphore = semaphore;
+ this.released = new AtomicBoolean(false);
+ }
+
+ public void release(){
+ if(this.released.compareAndSet(false, true)){
+ this.semaphore.release();
+ }
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java
new file mode 100644
index 0000000000..ca304646e4
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.future;
+
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+/**
+ * response future
+ */
+public class ResponseFuture {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(ResponseFuture.class);
+
+ private final static ConcurrentHashMap FUTURE_TABLE = new ConcurrentHashMap<>(256);
+
+ /**
+ * request unique identification
+ */
+ private final long opaque;
+
+ /**
+ * timeout
+ */
+ private final long timeoutMillis;
+
+ /**
+ * invokeCallback function
+ */
+ private final InvokeCallback invokeCallback;
+
+ /**
+ * releaseSemaphore
+ */
+ private final ReleaseSemaphore releaseSemaphore;
+
+ private final CountDownLatch latch = new CountDownLatch(1);
+
+ private final long beginTimestamp = System.currentTimeMillis();
+
+ /**
+ * response command
+ */
+ private volatile Command responseCommand;
+
+ private volatile boolean sendOk = true;
+
+ private volatile Throwable cause;
+
+ public ResponseFuture(long opaque, long timeoutMillis, InvokeCallback invokeCallback, ReleaseSemaphore releaseSemaphore) {
+ this.opaque = opaque;
+ this.timeoutMillis = timeoutMillis;
+ this.invokeCallback = invokeCallback;
+ this.releaseSemaphore = releaseSemaphore;
+ FUTURE_TABLE.put(opaque, this);
+ }
+
+ /**
+ * wait for response
+ *
+ * @return command
+ * @throws InterruptedException
+ */
+ public Command waitResponse() throws InterruptedException {
+ this.latch.await(timeoutMillis, TimeUnit.MILLISECONDS);
+ return this.responseCommand;
+ }
+
+ /**
+ * put response
+ *
+ * @param responseCommand responseCommand
+ */
+ public void putResponse(final Command responseCommand) {
+ this.responseCommand = responseCommand;
+ this.latch.countDown();
+ FUTURE_TABLE.remove(opaque);
+ }
+
+ public static ResponseFuture getFuture(long opaque){
+ return FUTURE_TABLE.get(opaque);
+ }
+
+ /**
+ * whether timeout
+ * @return timeout
+ */
+ public boolean isTimeout() {
+ long diff = System.currentTimeMillis() - this.beginTimestamp;
+ return diff > this.timeoutMillis;
+ }
+
+ /**
+ * execute invoke callback
+ */
+ public void executeInvokeCallback() {
+ if (invokeCallback != null) {
+ invokeCallback.operationComplete(this);
+ }
+ }
+
+ public boolean isSendOK() {
+ return sendOk;
+ }
+
+ public void setSendOk(boolean sendOk) {
+ this.sendOk = sendOk;
+ }
+
+ public void setCause(Throwable cause) {
+ this.cause = cause;
+ }
+
+ public Throwable getCause() {
+ return cause;
+ }
+
+ public long getOpaque() {
+ return opaque;
+ }
+
+ public long getTimeoutMillis() {
+ return timeoutMillis;
+ }
+
+ public long getBeginTimestamp() {
+ return beginTimestamp;
+ }
+
+ public Command getResponseCommand() {
+ return responseCommand;
+ }
+
+ public void setResponseCommand(Command responseCommand) {
+ this.responseCommand = responseCommand;
+ }
+
+ public InvokeCallback getInvokeCallback() {
+ return invokeCallback;
+ }
+
+ /**
+ * release
+ */
+ public void release() {
+ if(this.releaseSemaphore != null){
+ this.releaseSemaphore.release();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "ResponseFuture{" +
+ "opaque=" + opaque +
+ ", timeoutMillis=" + timeoutMillis +
+ ", invokeCallback=" + invokeCallback +
+ ", releaseSemaphore=" + releaseSemaphore +
+ ", latch=" + latch +
+ ", beginTimestamp=" + beginTimestamp +
+ ", responseCommand=" + responseCommand +
+ ", sendOk=" + sendOk +
+ ", cause=" + cause +
+ '}';
+ }
+
+ /**
+ * scan future table
+ */
+ public static void scanFutureTable(){
+ final List futureList = new LinkedList<>();
+ Iterator> it = FUTURE_TABLE.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry next = it.next();
+ ResponseFuture future = next.getValue();
+ if ((future.getBeginTimestamp() + future.getTimeoutMillis() + 1000) <= System.currentTimeMillis()) {
+ futureList.add(future);
+ it.remove();
+ LOGGER.warn("remove timeout request : {}", future);
+ }
+ }
+ for (ResponseFuture future : futureList) {
+ try {
+ future.release();
+ future.executeInvokeCallback();
+ } catch (Throwable ex) {
+ LOGGER.warn("scanFutureTable, execute callback error", ex);
+ }
+ }
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java
new file mode 100644
index 0000000000..d5d0d4df83
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.remote.handler;
+
+import io.netty.channel.*;
+import org.apache.dolphinscheduler.remote.NettyRemotingClient;
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.future.ResponseFuture;
+import org.apache.dolphinscheduler.remote.utils.ChannelUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutorService;
+
+/**
+ * netty client request handler
+ */
+@ChannelHandler.Sharable
+public class NettyClientHandler extends ChannelInboundHandlerAdapter {
+
+ private final Logger logger = LoggerFactory.getLogger(NettyClientHandler.class);
+
+ /**
+ * netty client
+ */
+ private final NettyRemotingClient nettyRemotingClient;
+
+ /**
+ * callback thread executor
+ */
+ private final ExecutorService callbackExecutor;
+
+ public NettyClientHandler(NettyRemotingClient nettyRemotingClient, ExecutorService callbackExecutor){
+ this.nettyRemotingClient = nettyRemotingClient;
+ this.callbackExecutor = callbackExecutor;
+ }
+
+ /**
+ * When the current channel is not active,
+ * the current channel has reached the end of its life cycle
+ *
+ * @param ctx channel handler context
+ * @throws Exception
+ */
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx) throws Exception {
+ nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel()));
+ ctx.channel().close();
+ }
+
+ /**
+ * The current channel reads data from the remote
+ *
+ * @param ctx channel handler context
+ * @param msg message
+ * @throws Exception
+ */
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
+ processReceived((Command)msg);
+ }
+
+ /**
+ * process received logic
+ *
+ * @param responseCommand responseCommand
+ */
+ private void processReceived(final Command responseCommand) {
+ ResponseFuture future = ResponseFuture.getFuture(responseCommand.getOpaque());
+ if(future != null){
+ future.setResponseCommand(responseCommand);
+ future.release();
+ if(future.getInvokeCallback() != null){
+ this.callbackExecutor.submit(new Runnable() {
+ @Override
+ public void run() {
+ future.executeInvokeCallback();
+ }
+ });
+ } else{
+ future.putResponse(responseCommand);
+ }
+ } else{
+ logger.warn("receive response {}, but not matched any request ", responseCommand);
+ }
+ }
+
+ /**
+ * caught exception
+ * @param ctx channel handler context
+ * @param cause cause
+ * @throws Exception
+ */
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+ logger.error("exceptionCaught : {}", cause);
+ nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel()));
+ ctx.channel().close();
+ }
+
+ /**
+ * channel write changed
+ *
+ * @param ctx channel handler context
+ * @throws Exception
+ */
+ @Override
+ public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception {
+ Channel ch = ctx.channel();
+ ChannelConfig config = ch.config();
+
+ if (!ch.isWritable()) {
+ if (logger.isWarnEnabled()) {
+ logger.warn("{} is not writable, over high water level : {}",
+ new Object[]{ch, config.getWriteBufferHighWaterMark()});
+ }
+
+ config.setAutoRead(false);
+ } else {
+ if (logger.isWarnEnabled()) {
+ logger.warn("{} is writable, to low water : {}",
+ new Object[]{ch, config.getWriteBufferLowWaterMark()});
+ }
+ config.setAutoRead(true);
+ }
+ }
+}
\ No newline at end of file
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java
new file mode 100644
index 0000000000..eabd6560de
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.handler;
+
+import io.netty.channel.*;
+import org.apache.dolphinscheduler.remote.NettyRemotingServer;
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
+import org.apache.dolphinscheduler.remote.utils.ChannelUtils;
+import org.apache.dolphinscheduler.remote.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+
+/**
+ * netty server request handler
+ */
+@ChannelHandler.Sharable
+public class NettyServerHandler extends ChannelInboundHandlerAdapter {
+
+ private final Logger logger = LoggerFactory.getLogger(NettyServerHandler.class);
+
+ /**
+ * netty remote server
+ */
+ private final NettyRemotingServer nettyRemotingServer;
+
+ /**
+ * server processors queue
+ */
+ private final ConcurrentHashMap> processors = new ConcurrentHashMap();
+
+ public NettyServerHandler(NettyRemotingServer nettyRemotingServer){
+ this.nettyRemotingServer = nettyRemotingServer;
+ }
+
+ /**
+ * When the current channel is not active,
+ * the current channel has reached the end of its life cycle
+ * @param ctx channel handler context
+ * @throws Exception
+ */
+ @Override
+ public void channelInactive(ChannelHandlerContext ctx) throws Exception {
+ ctx.channel().close();
+ }
+
+ /**
+ * The current channel reads data from the remote end
+ *
+ * @param ctx channel handler context
+ * @param msg message
+ * @throws Exception
+ */
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
+ processReceived(ctx.channel(), (Command)msg);
+ }
+
+ /**
+ * register processor
+ *
+ * @param commandType command type
+ * @param processor processor
+ */
+ public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor) {
+ this.registerProcessor(commandType, processor, null);
+ }
+
+ /**
+ * register processor
+ *
+ * @param commandType command type
+ * @param processor processor
+ * @param executor thread executor
+ */
+ public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) {
+ ExecutorService executorRef = executor;
+ if(executorRef == null){
+ executorRef = nettyRemotingServer.getDefaultExecutor();
+ }
+ this.processors.putIfAbsent(commandType, new Pair(processor, executorRef));
+ }
+
+ /**
+ * process received logic
+ * @param channel channel
+ * @param msg message
+ */
+ private void processReceived(final Channel channel, final Command msg) {
+ final CommandType commandType = msg.getType();
+ final Pair pair = processors.get(commandType);
+ if (pair != null) {
+ Runnable r = new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ pair.getLeft().process(channel, msg);
+ } catch (Throwable ex) {
+ logger.error("process msg {} error : {}", msg, ex);
+ }
+ }
+ };
+ try {
+ pair.getRight().submit(r);
+ } catch (RejectedExecutionException e) {
+ logger.warn("thread pool is full, discard msg {} from {}", msg, ChannelUtils.getRemoteAddress(channel));
+ }
+ } else {
+ logger.warn("commandType {} not support", commandType);
+ }
+ }
+
+ /**
+ * caught exception
+ *
+ * @param ctx channel handler context
+ * @param cause cause
+ * @throws Exception
+ */
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+ logger.error("exceptionCaught : {}", cause);
+ ctx.channel().close();
+ }
+
+ /**
+ * channel write changed
+ *
+ * @param ctx channel handler context
+ * @throws Exception
+ */
+ @Override
+ public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception {
+ Channel ch = ctx.channel();
+ ChannelConfig config = ch.config();
+
+ if (!ch.isWritable()) {
+ if (logger.isWarnEnabled()) {
+ logger.warn("{} is not writable, over high water level : {}",
+ new Object[]{ch, config.getWriteBufferHighWaterMark()});
+ }
+
+ config.setAutoRead(false);
+ } else {
+ if (logger.isWarnEnabled()) {
+ logger.warn("{} is writable, to low water : {}",
+ new Object[]{ch, config.getWriteBufferLowWaterMark()});
+ }
+ config.setAutoRead(true);
+ }
+ }
+}
\ No newline at end of file
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java
new file mode 100644
index 0000000000..6966b53d17
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.remote.processor;
+
+import io.netty.channel.Channel;
+import org.apache.dolphinscheduler.remote.command.Command;
+
+/**
+ * netty request processor
+ */
+public interface NettyRequestProcessor {
+
+ /**
+ * process logic
+ * @param channel channel
+ * @param command command
+ */
+ void process(final Channel channel, final Command command);
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Address.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Address.java
new file mode 100644
index 0000000000..f61dcd615c
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Address.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.remote.utils;
+
+import java.io.Serializable;
+
+/**
+ * server address
+ */
+public class Address implements Serializable {
+
+ /**
+ * host
+ */
+ private String host;
+
+ /**
+ * port
+ */
+ private int port;
+
+ public Address(){
+ //NOP
+ }
+
+ public Address(String host, int port){
+ this.host = host;
+ this.port = port;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public void setHost(String host) {
+ this.host = host;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public void setPort(int port) {
+ this.port = port;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((host == null) ? 0 : host.hashCode());
+ result = prime * result + port;
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ Address other = (Address) obj;
+ if (host == null) {
+ if (other.host != null) {
+ return false;
+ }
+ } else if (!host.equals(other.host)) {
+ return false;
+ }
+ return port == other.port;
+ }
+
+ @Override
+ public String toString() {
+ return "Address [host=" + host + ", port=" + port + "]";
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java
new file mode 100644
index 0000000000..048ea86acb
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * caller thread execute
+ */
+public class CallerThreadExecutePolicy implements RejectedExecutionHandler {
+
+ private final Logger logger = LoggerFactory.getLogger(CallerThreadExecutePolicy.class);
+
+ @Override
+ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
+ logger.warn("queue is full, trigger caller thread execute");
+ r.run();
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java
new file mode 100644
index 0000000000..d7af5fe165
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.remote.utils;
+
+import io.netty.channel.Channel;
+
+import java.net.InetSocketAddress;
+
+/**
+ * channel utils
+ */
+public class ChannelUtils {
+
+ /**
+ * get local address
+ *
+ * @param channel channel
+ * @return local address
+ */
+ public static String getLocalAddress(Channel channel){
+ return ((InetSocketAddress)channel.localAddress()).getAddress().getHostAddress();
+ }
+
+ /**
+ * get remote address
+ * @param channel channel
+ * @return remote address
+ */
+ public static String getRemoteAddress(Channel channel){
+ return ((InetSocketAddress)channel.remoteAddress()).getAddress().getHostAddress();
+ }
+
+ /**
+ * channel to address
+ * @param channel channel
+ * @return address
+ */
+ public static Address toAddress(Channel channel){
+ InetSocketAddress socketAddress = ((InetSocketAddress)channel.remoteAddress());
+ return new Address(socketAddress.getAddress().getHostAddress(), socketAddress.getPort());
+ }
+
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java
new file mode 100644
index 0000000000..5733b17790
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.remote.utils;
+
+import java.nio.charset.Charset;
+
+
+/**
+ * constant
+ */
+public class Constants {
+
+ public static final String COMMA = ",";
+
+ public static final String SLASH = "/";
+
+ /**
+ * charset
+ */
+ public static final Charset UTF8 = Charset.forName("UTF-8");
+
+ /**
+ * cpus
+ */
+ public static final int CPUS = Runtime.getRuntime().availableProcessors();
+
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java
new file mode 100644
index 0000000000..e96796a05c
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/FastJsonSerializer.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.remote.utils;
+
+import com.alibaba.fastjson.JSON;
+
+/**
+ * json serialize or deserialize
+ */
+public class FastJsonSerializer {
+
+ /**
+ * serialize to byte
+ *
+ * @param obj object
+ * @param object type
+ * @return byte array
+ */
+ public static byte[] serialize(T obj) {
+ String json = JSON.toJSONString(obj);
+ return json.getBytes(Constants.UTF8);
+ }
+
+ /**
+ * serialize to string
+ * @param obj object
+ * @param object type
+ * @return string
+ */
+ public static String serializeToString(T obj) {
+ return JSON.toJSONString(obj);
+ }
+
+ /**
+ * deserialize
+ *
+ * @param src byte array
+ * @param clazz class
+ * @param deserialize type
+ * @return deserialize type
+ */
+ public static T deserialize(byte[] src, Class clazz) {
+ return JSON.parseObject(new String(src, Constants.UTF8), clazz);
+ }
+
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java
new file mode 100644
index 0000000000..2f0d05ebd4
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.remote.utils;
+
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * thread factory
+ */
+public class NamedThreadFactory implements ThreadFactory {
+
+ private final AtomicInteger increment = new AtomicInteger(1);
+
+ /**
+ * name
+ */
+ private final String name;
+
+ /**
+ * count
+ */
+ private final int count;
+
+ public NamedThreadFactory(String name){
+ this(name, 0);
+ }
+
+ public NamedThreadFactory(String name, int count){
+ this.name = name;
+ this.count = count;
+ }
+
+ /**
+ * create thread
+ * @param r runnable
+ * @return thread
+ */
+ @Override
+ public Thread newThread(Runnable r) {
+ final String threadName = count > 0 ? String.format(name + "_%d_%d", count, increment.getAndIncrement())
+ : String.format(name + "_%d", increment.getAndIncrement());
+ Thread t = new Thread(r, threadName);
+ t.setDaemon(true);
+ return t;
+ }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java
new file mode 100644
index 0000000000..2042191486
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.utils;
+
+
+/**
+ * key value pair
+ *
+ * @param L generic type
+ * @param R generic type
+ */
+public class Pair {
+
+ private L left;
+
+ private R right;
+
+ public Pair(L left, R right) {
+ this.left = left;
+ this.right = right;
+ }
+
+ public L getLeft() {
+ return left;
+ }
+
+ public void setLeft(L left) {
+ this.left = left;
+ }
+
+ public R getRight() {
+ return right;
+ }
+
+ public void setRight(R right) {
+ this.right = right;
+ }
+}
diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/FastJsonSerializerTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/FastJsonSerializerTest.java
new file mode 100644
index 0000000000..97166cca70
--- /dev/null
+++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/FastJsonSerializerTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote;
+
+
+import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class FastJsonSerializerTest {
+
+ @Test
+ public void testSerialize(){
+ TestObj testObj = new TestObj();
+ testObj.setAge(12);
+ byte[] serializeByte = FastJsonSerializer.serialize(testObj);
+
+ //
+ TestObj deserialize = FastJsonSerializer.deserialize(serializeByte, TestObj.class);
+
+ Assert.assertEquals(testObj.getAge(), deserialize.getAge());
+ }
+
+ static class TestObj {
+
+ private int age;
+
+ public int getAge() {
+ return age;
+ }
+
+ public void setAge(int age) {
+ this.age = age;
+ }
+
+ @Override
+ public String toString() {
+ return "TestObj{" +
+ "age=" + age +
+ '}';
+ }
+ }
+}
diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java
new file mode 100644
index 0000000000..ef46c2c781
--- /dev/null
+++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote;
+
+import io.netty.channel.Channel;
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.command.Ping;
+import org.apache.dolphinscheduler.remote.command.Pong;
+import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
+import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
+import org.apache.dolphinscheduler.remote.future.InvokeCallback;
+import org.apache.dolphinscheduler.remote.future.ResponseFuture;
+import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
+import org.apache.dolphinscheduler.remote.utils.Address;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * netty remote client test
+ */
+public class NettyRemotingClientTest {
+
+
+ /**
+ * test sned sync
+ */
+ @Test
+ public void testSendSync(){
+ NettyServerConfig serverConfig = new NettyServerConfig();
+
+ NettyRemotingServer server = new NettyRemotingServer(serverConfig);
+ server.registerProcessor(CommandType.PING, new NettyRequestProcessor() {
+ @Override
+ public void process(Channel channel, Command command) {
+ channel.writeAndFlush(Pong.create(command.getOpaque()));
+ }
+ });
+
+
+ server.start();
+ //
+ final NettyClientConfig clientConfig = new NettyClientConfig();
+ NettyRemotingClient client = new NettyRemotingClient(clientConfig);
+ Command commandPing = Ping.create();
+ try {
+ Command response = client.sendSync(new Address("127.0.0.1", serverConfig.getListenPort()), commandPing, 2000);
+ Assert.assertEquals(commandPing.getOpaque(), response.getOpaque());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ server.close();
+ client.close();
+ }
+
+ /**
+ * test sned async
+ */
+ @Test
+ public void testSendAsync(){
+ NettyServerConfig serverConfig = new NettyServerConfig();
+
+ NettyRemotingServer server = new NettyRemotingServer(serverConfig);
+ server.registerProcessor(CommandType.PING, new NettyRequestProcessor() {
+ @Override
+ public void process(Channel channel, Command command) {
+ channel.writeAndFlush(Pong.create(command.getOpaque()));
+ }
+ });
+ server.start();
+ //
+ final NettyClientConfig clientConfig = new NettyClientConfig();
+ NettyRemotingClient client = new NettyRemotingClient(clientConfig);
+ CountDownLatch latch = new CountDownLatch(1);
+ Command commandPing = Ping.create();
+ try {
+ final AtomicLong opaque = new AtomicLong(0);
+ client.sendAsync(new Address("127.0.0.1", serverConfig.getListenPort()), commandPing, 2000, new InvokeCallback() {
+ @Override
+ public void operationComplete(ResponseFuture responseFuture) {
+ opaque.set(responseFuture.getOpaque());
+ latch.countDown();
+ }
+ });
+ latch.await();
+ Assert.assertEquals(commandPing.getOpaque(), opaque.get());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ server.close();
+ client.close();
+ }
+}
diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/ResponseFutureTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/ResponseFutureTest.java
new file mode 100644
index 0000000000..8836043257
--- /dev/null
+++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/ResponseFutureTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote;
+
+
+import org.apache.dolphinscheduler.remote.future.InvokeCallback;
+import org.apache.dolphinscheduler.remote.future.ResponseFuture;
+import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class ResponseFutureTest {
+
+ @Test
+ public void testScanFutureTable(){
+ ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("executor-service"));
+ executorService.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ ResponseFuture.scanFutureTable();
+ }
+ }, 3000, 1000, TimeUnit.MILLISECONDS);
+
+ CountDownLatch latch = new CountDownLatch(1);
+ InvokeCallback invokeCallback = new InvokeCallback() {
+ @Override
+ public void operationComplete(ResponseFuture responseFuture) {
+ latch.countDown();
+ }
+ };
+ ResponseFuture future = new ResponseFuture(1, 2000, invokeCallback, null);
+ try {
+ latch.await(5000, TimeUnit.MILLISECONDS);
+ Assert.assertTrue(ResponseFuture.getFuture(1) == null);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ executorService.shutdownNow();
+ }
+}
diff --git a/dolphinscheduler-rpc/pom.xml b/dolphinscheduler-rpc/pom.xml
deleted file mode 100644
index 680a4a24c0..0000000000
--- a/dolphinscheduler-rpc/pom.xml
+++ /dev/null
@@ -1,113 +0,0 @@
-
-
-
-
-
- org.apache.dolphinscheduler
- dolphinscheduler
- 1.2.1-SNAPSHOT
-
- 4.0.0
-
- dolphinscheduler-rpc
-
- dolphinscheduler-rpc
- https://github.com/apache/incubator-dolphinscheduler
-
-
- UTF-8
- 1.8
- 1.8
-
- 3.5.1
- 1.9.0
-
-
-
-
- com.google.protobuf
- protobuf-java
- ${protobuf.version}
-
-
- io.grpc
- grpc-netty
- ${grpc.version}
-
-
- io.grpc
- grpc-protobuf
- ${grpc.version}
-
-
- io.grpc
- grpc-stub
- ${grpc.version}
-
-
-
- com.google.guava
- guava
-
-
-
-
-
-
- kr.motd.maven
- os-maven-plugin
- 1.5.0.Final
-
-
-
-
- org.xolstice.maven.plugins
- protobuf-maven-plugin
- 0.5.0
-
- com.google.protobuf:protoc:3.5.1-1:exe:${os.detected.classifier}
- grpc-java
- io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}
-
-
-
- compile
-
- compile
-
-
-
- compile-custom
-
- compile-custom
-
-
-
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
-
- ${java.version}
- ${java.version}
- ${project.build.sourceEncoding}
-
-
-
-
-
diff --git a/dolphinscheduler-rpc/src/main/proto/scheduler.proto b/dolphinscheduler-rpc/src/main/proto/scheduler.proto
deleted file mode 100644
index b8b595cb2a..0000000000
--- a/dolphinscheduler-rpc/src/main/proto/scheduler.proto
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-syntax = "proto3";
-
-package schduler;
-
-option java_multiple_files = true;
-option java_package = "org.apache.dolphinscheduler.rpc";
-option java_outer_classname = "SchdulerProto";
-
-
-/**
- * return str info
- */
-message RetStrInfo {
- /**
- * str msg info
- */
- string msg = 1 ;
-}
-
-/**
- * return byte info
- */
-message RetByteInfo {
- /**
- * byte data info
- */
- bytes data = 1;
-}
-
-/**
- * log parameter
- */
-message LogParameter {
-
- /**
- * path
- */
- string path = 1 ;
-
- /**
- * skip line num
- */
- int32 skipLineNum = 2 ;
-
- /**
- * display limt num
- */
- int32 limit = 3 ;
-}
-
-
-/**
- * path parameter
- */
-message PathParameter {
-
- /**
- * path
- */
- string path = 1 ;
-}
-
-/**
- * log view service
- */
-service LogViewService {
-
- /**
- * roll view log
- */
- rpc rollViewLog(LogParameter) returns (RetStrInfo) {};
-
- /**
- * view all log
- */
- rpc viewLog(PathParameter) returns (RetStrInfo) {};
-
- /**
- * get log bytes
- */
- rpc getLogBytes(PathParameter) returns (RetByteInfo) {};
-}
-
diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml
index 751fd919a8..080b87ebaa 100644
--- a/dolphinscheduler-server/pom.xml
+++ b/dolphinscheduler-server/pom.xml
@@ -71,7 +71,7 @@
org.apache.dolphinscheduler
- dolphinscheduler-rpc
+ dolphinscheduler-service
org.apache.curator
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
new file mode 100644
index 0000000000..4e4404ea1c
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.log;
+
+import io.netty.channel.Channel;
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.command.log.*;
+import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
+import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * logger request process logic
+ */
+public class LoggerRequestProcessor implements NettyRequestProcessor {
+
+ private final Logger logger = LoggerFactory.getLogger(LoggerRequestProcessor.class);
+
+ private final ThreadPoolExecutor executor;
+
+ public LoggerRequestProcessor(){
+ this.executor = new ThreadPoolExecutor(4, 4, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100));
+ }
+
+ @Override
+ public void process(Channel channel, Command command) {
+ logger.info("received command : {}", command);
+
+ /**
+ * reuqest task log command type
+ */
+ final CommandType commandType = command.getType();
+ switch (commandType){
+ case GET_LOG_BYTES_REQUEST:
+ GetLogBytesRequestCommand getLogRequest = FastJsonSerializer.deserialize(
+ command.getBody(), GetLogBytesRequestCommand.class);
+ byte[] bytes = getFileContentBytes(getLogRequest.getPath());
+ GetLogBytesResponseCommand getLogResponse = new GetLogBytesResponseCommand(bytes);
+ channel.writeAndFlush(getLogResponse.convert2Command(command.getOpaque()));
+ break;
+ case VIEW_WHOLE_LOG_REQUEST:
+ ViewLogRequestCommand viewLogRequest = FastJsonSerializer.deserialize(
+ command.getBody(), ViewLogRequestCommand.class);
+ String msg = readWholeFileContent(viewLogRequest.getPath());
+ ViewLogResponseCommand viewLogResponse = new ViewLogResponseCommand(msg);
+ channel.writeAndFlush(viewLogResponse.convert2Command(command.getOpaque()));
+ break;
+ case ROLL_VIEW_LOG_REQUEST:
+ RollViewLogRequestCommand rollViewLogRequest = FastJsonSerializer.deserialize(
+ command.getBody(), RollViewLogRequestCommand.class);
+ List lines = readPartFileContent(rollViewLogRequest.getPath(),
+ rollViewLogRequest.getSkipLineNum(), rollViewLogRequest.getLimit());
+ StringBuilder builder = new StringBuilder();
+ for (String line : lines){
+ builder.append(line + "\r\n");
+ }
+ RollViewLogResponseCommand rollViewLogRequestResponse = new RollViewLogResponseCommand(builder.toString());
+ channel.writeAndFlush(rollViewLogRequestResponse.convert2Command(command.getOpaque()));
+ break;
+ default:
+ throw new IllegalArgumentException("unknown commandType");
+ }
+ }
+
+ public ExecutorService getExecutor(){
+ return this.executor;
+ }
+
+ /**
+ * get files content bytes，for down load file
+ *
+ * @param filePath file path
+ * @return byte array of file
+ * @throws Exception exception
+ */
+ private byte[] getFileContentBytes(String filePath){
+ InputStream in = null;
+ ByteArrayOutputStream bos = null;
+ try {
+ in = new FileInputStream(filePath);
+ bos = new ByteArrayOutputStream();
+ byte[] buf = new byte[1024];
+ int len;
+ while ((len = in.read(buf)) != -1) {
+ bos.write(buf, 0, len);
+ }
+ return bos.toByteArray();
+ }catch (IOException e){
+ logger.error("get file bytes error",e);
+ }finally {
+ if (bos != null){
+ try {
+ bos.close();
+ } catch (IOException ignore) {}
+ }
+ if (in != null){
+ try {
+ in.close();
+ } catch (IOException ignore) {}
+ }
+ }
+ return new byte[0];
+ }
+
+ /**
+ * read part file content，can skip any line and read some lines
+ *
+ * @param filePath file path
+ * @param skipLine skip line
+ * @param limit read lines limit
+ * @return part file content
+ */
+ private List readPartFileContent(String filePath,
+ int skipLine,
+ int limit){
+ try (Stream stream = Files.lines(Paths.get(filePath))) {
+ return stream.skip(skipLine).limit(limit).collect(Collectors.toList());
+ } catch (IOException e) {
+ logger.error("read file error",e);
+ }
+ return Collections.EMPTY_LIST;
+ }
+
+ /**
+ * read whole file content
+ *
+ * @param filePath file path
+ * @return whole file content
+ */
+ private String readWholeFileContent(String filePath){
+ BufferedReader br = null;
+ String line;
+ StringBuilder sb = new StringBuilder();
+ try {
+ br = new BufferedReader(new InputStreamReader(new FileInputStream(filePath)));
+ while ((line = br.readLine()) != null){
+ sb.append(line + "\r\n");
+ }
+ return sb.toString();
+ }catch (IOException e){
+ logger.error("read file error",e);
+ }finally {
+ try {
+ if (br != null){
+ br.close();
+ }
+ } catch (IOException ignore) {}
+ }
+ return "";
+ }
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java
new file mode 100644
index 0000000000..3520fb09ec
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerServer.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.log;
+
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.remote.NettyRemotingServer;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * logger server
+ */
+public class LoggerServer {
+
+ private static final Logger logger = LoggerFactory.getLogger(LoggerServer.class);
+
+ /**
+ * netty server
+ */
+ private final NettyRemotingServer server;
+
+ /**
+ * netty server config
+ */
+ private final NettyServerConfig serverConfig;
+
+ /**
+ * loggger request processor
+ */
+ private final LoggerRequestProcessor requestProcessor;
+
+ public LoggerServer(){
+ this.serverConfig = new NettyServerConfig();
+ this.serverConfig.setListenPort(Constants.RPC_PORT);
+ this.server = new NettyRemotingServer(serverConfig);
+ this.requestProcessor = new LoggerRequestProcessor();
+ this.server.registerProcessor(CommandType.GET_LOG_BYTES_REQUEST, requestProcessor, requestProcessor.getExecutor());
+ this.server.registerProcessor(CommandType.ROLL_VIEW_LOG_REQUEST, requestProcessor, requestProcessor.getExecutor());
+ this.server.registerProcessor(CommandType.VIEW_WHOLE_LOG_REQUEST, requestProcessor, requestProcessor.getExecutor());
+ }
+
+ /**
+ * main launches the server from the command line.
+ * @param args arguments
+ */
+ public static void main(String[] args) {
+ final LoggerServer server = new LoggerServer();
+ server.start();
+ }
+
+ /**
+ * server start
+ */
+ public void start() {
+ this.server.start();
+ logger.info("logger server started, listening on port : {}" , Constants.RPC_PORT);
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ LoggerServer.this.stop();
+ }
+ });
+ }
+
+ /**
+ * stop
+ */
+ public void stop() {
+ this.server.close();
+ logger.info("logger server shut down");
+ }
+
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
index 0647b9450b..6b5063cba4 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
@@ -22,14 +22,14 @@ import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.common.utils.StringUtils;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerThread;
-import org.apache.dolphinscheduler.dao.quartz.ProcessScheduleJob;
-import org.apache.dolphinscheduler.dao.quartz.QuartzExecutors;
import org.apache.dolphinscheduler.server.zk.ZKMasterClient;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.service.quartz.ProcessScheduleJob;
+import org.apache.dolphinscheduler.service.quartz.QuartzExecutors;
import org.quartz.SchedulerException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,10 +66,10 @@ public class MasterServer implements IStoppable {
private ScheduledExecutorService heartbeatMasterService;
/**
- * dolphinscheduler database interface
+ * process service
*/
@Autowired
- protected ProcessDao processDao;
+ protected ProcessService processService;
/**
* master exec thread pool
@@ -77,17 +77,18 @@ public class MasterServer implements IStoppable {
private ExecutorService masterSchedulerService;
/**
- * spring application context
- * only use it for initialization
+ * master config
*/
@Autowired
- private SpringApplicationContext springApplicationContext;
+ private MasterConfig masterConfig;
+
/**
- * master config
+ * spring application context
+ * only use it for initialization
*/
@Autowired
- private MasterConfig masterConfig;
+ private SpringApplicationContext springApplicationContext;
/**
@@ -111,7 +112,7 @@ public class MasterServer implements IStoppable {
masterSchedulerService = ThreadUtils.newDaemonSingleThreadExecutor("Master-Scheduler-Thread");
- heartbeatMasterService = ThreadUtils.newDaemonThreadScheduledExecutor("Master-Main-Thread",Constants.defaulMasterHeartbeatThreadNum);
+ heartbeatMasterService = ThreadUtils.newDaemonThreadScheduledExecutor("Master-Main-Thread",Constants.DEFAULT_MASTER_HEARTBEAT_THREAD_NUM);
// heartbeat thread implement
Runnable heartBeatThread = heartBeatThread();
@@ -126,7 +127,7 @@ public class MasterServer implements IStoppable {
// master scheduler thread
MasterSchedulerThread masterSchedulerThread = new MasterSchedulerThread(
zkMasterClient,
- processDao,
+ processService,
masterConfig.getMasterExecThreads());
// submit master scheduler thread
@@ -136,7 +137,7 @@ public class MasterServer implements IStoppable {
// what system should do if exception
try {
logger.info("start Quartz server...");
- ProcessScheduleJob.init(processDao);
+ ProcessScheduleJob.init(processService);
QuartzExecutors.getInstance().start();
} catch (Exception e) {
try {
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java
index c1552c4621..f8fcb1456d 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java
@@ -16,15 +16,15 @@
*/
package org.apache.dolphinscheduler.server.master.runner;
-import org.apache.dolphinscheduler.common.queue.ITaskQueue;
-import org.apache.dolphinscheduler.common.queue.TaskQueueFactory;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.dao.AlertDao;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.utils.BeanContext;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.service.queue.ITaskQueue;
+import org.apache.dolphinscheduler.service.queue.TaskQueueFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,9 +41,9 @@ public class MasterBaseTaskExecThread implements Callable {
private static final Logger logger = LoggerFactory.getLogger(MasterBaseTaskExecThread.class);
/**
- * process dao
+ * process service
*/
- protected ProcessDao processDao;
+ protected ProcessService processService;
/**
* alert database access
@@ -81,7 +81,7 @@ public class MasterBaseTaskExecThread implements Callable {
* @param processInstance process instance
*/
public MasterBaseTaskExecThread(TaskInstance taskInstance, ProcessInstance processInstance){
- this.processDao = BeanContext.getBean(ProcessDao.class);
+ this.processService = BeanContext.getBean(ProcessService.class);
this.alertDao = BeanContext.getBean(AlertDao.class);
this.processInstance = processInstance;
this.taskQueue = TaskQueueFactory.getTaskQueueInstance();
@@ -121,14 +121,14 @@ public class MasterBaseTaskExecThread implements Callable {
try {
if(!submitDB){
// submit task to db
- task = processDao.submitTask(taskInstance, processInstance);
+ task = processService.submitTask(taskInstance, processInstance);
if(task != null && task.getId() != 0){
submitDB = true;
}
}
if(submitDB && !submitQueue){
// submit task to queue
- submitQueue = processDao.submitTaskToQueue(task);
+ submitQueue = processService.submitTaskToQueue(task);
}
if(submitDB && submitQueue){
return task;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
index 2b1ff4d23f..4b22b27bec 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
@@ -25,17 +25,19 @@ import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
import org.apache.dolphinscheduler.common.process.ProcessDag;
+import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.*;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.utils.DagHelper;
-import org.apache.dolphinscheduler.dao.utils.cron.CronUtils;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.utils.AlertManager;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -108,6 +110,11 @@ public class MasterExecThread implements Runnable {
*/
private Map forbiddenTaskList = new ConcurrentHashMap<>();
+ /**
+ * skip task map
+ */
+ private Map skipTaskNodeList = new ConcurrentHashMap<>();
+
/**
* recover tolerance fault task list
*/
@@ -124,9 +131,9 @@ public class MasterExecThread implements Runnable {
private DAG dag;
/**
- * process dao
+ * process service
*/
- private ProcessDao processDao;
+ private ProcessService processService;
/**
* master config
@@ -136,10 +143,10 @@ public class MasterExecThread implements Runnable {
/**
* constructor of MasterExecThread
* @param processInstance process instance
- * @param processDao process dao
+ * @param processService process dao
*/
- public MasterExecThread(ProcessInstance processInstance,ProcessDao processDao){
- this.processDao = processDao;
+ public MasterExecThread(ProcessInstance processInstance, ProcessService processService){
+ this.processService = processService;
this.processInstance = processInstance;
this.masterConfig = SpringApplicationContext.getBean(MasterConfig.class);
@@ -177,7 +184,7 @@ public class MasterExecThread implements Runnable {
logger.error("process execute failed, process id:{}", processInstance.getId());
processInstance.setState(ExecutionStatus.FAILURE);
processInstance.setEndTime(new Date());
- processDao.updateProcessInstance(processInstance);
+ processService.updateProcessInstance(processInstance);
}finally {
taskExecService.shutdown();
// post handle
@@ -205,11 +212,11 @@ public class MasterExecThread implements Runnable {
Date startDate = DateUtils.getScheduleDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE));
Date endDate = DateUtils.getScheduleDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE));
- processDao.saveProcessInstance(processInstance);
+ processService.saveProcessInstance(processInstance);
// get schedules
int processDefinitionId = processInstance.getProcessDefinitionId();
- List schedules = processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId);
+ List schedules = processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId);
List listDate = Lists.newLinkedList();
if(!CollectionUtils.isEmpty(schedules)){
for (Schedule schedule : schedules) {
@@ -223,7 +230,7 @@ public class MasterExecThread implements Runnable {
iterator = listDate.iterator();
scheduleDate = iterator.next();
processInstance.setScheduleTime(scheduleDate);
- processDao.updateProcessInstance(processInstance);
+ processService.updateProcessInstance(processInstance);
}else{
scheduleDate = processInstance.getScheduleTime();
if(scheduleDate == null){
@@ -239,7 +246,7 @@ public class MasterExecThread implements Runnable {
logger.error("process {} dag is null, please check out parameters",
processInstance.getId());
processInstance.setState(ExecutionStatus.SUCCESS);
- processDao.updateProcessInstance(processInstance);
+ processService.updateProcessInstance(processInstance);
return;
}
@@ -281,10 +288,10 @@ public class MasterExecThread implements Runnable {
processInstance.setCommandParam(JSONUtils.toJson(cmdParam));
}
- List taskInstanceList = processDao.findValidTaskListByProcessId(processInstance.getId());
+ List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId());
for(TaskInstance taskInstance : taskInstanceList){
taskInstance.setFlag(Flag.NO);
- processDao.updateTaskInstance(taskInstance);
+ processService.updateTaskInstance(taskInstance);
}
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(
@@ -292,7 +299,7 @@ public class MasterExecThread implements Runnable {
processInstance.getProcessDefinition().getGlobalParamList(),
CommandType.COMPLEMENT_DATA, processInstance.getScheduleTime()));
- processDao.saveProcessInstance(processInstance);
+ processService.saveProcessInstance(processInstance);
}
// flow end
@@ -320,11 +327,11 @@ public class MasterExecThread implements Runnable {
*/
private void endProcess() {
processInstance.setEndTime(new Date());
- processDao.updateProcessInstance(processInstance);
+ processService.updateProcessInstance(processInstance);
if(processInstance.getState().typeIsWaittingThread()){
- processDao.createRecoveryWaitingThreadCommand(null, processInstance);
+ processService.createRecoveryWaitingThreadCommand(null, processInstance);
}
- List taskInstances = processDao.findValidTaskListByProcessId(processInstance.getId());
+ List taskInstances = processService.findValidTaskListByProcessId(processInstance.getId());
alertManager.sendAlertProcessInstance(processInstance, taskInstances);
}
@@ -361,7 +368,7 @@ public class MasterExecThread implements Runnable {
dependFailedTask.clear();
completeTaskList.clear();
errorTaskList.clear();
- List taskInstanceList = processDao.findValidTaskListByProcessId(processInstance.getId());
+ List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId());
for(TaskInstance task : taskInstanceList){
if(task.isTaskComplete()){
completeTaskList.put(task.getName(), task);
@@ -417,7 +424,7 @@ public class MasterExecThread implements Runnable {
* @return TaskInstance
*/
private TaskInstance findTaskIfExists(String taskName){
- List taskInstanceList = processDao.findValidTaskListByProcessId(this.processInstance.getId());
+ List taskInstanceList = processService.findValidTaskListByProcessId(this.processInstance.getId());
for(TaskInstance taskInstance : taskInstanceList){
if(taskInstance.getName().equals(taskName)){
return taskInstance;
@@ -433,7 +440,7 @@ public class MasterExecThread implements Runnable {
* @return TaskInstance
*/
private TaskInstance createTaskInstance(ProcessInstance processInstance, String nodeName,
- TaskNode taskNode, String parentNodeName) {
+ TaskNode taskNode) {
TaskInstance taskInstance = findTaskIfExists(nodeName);
if(taskInstance == null){
@@ -483,58 +490,140 @@ public class MasterExecThread implements Runnable {
}
/**
- * get post task instance by node
- * @param dag dag
- * @param parentNodeName parent node name
- * @return task instance list
+ * is there have conditions after the parent node
+ * @param parentNodeName
+ * @return
*/
- private List getPostTaskInstanceByNode(DAG dag, String parentNodeName){
+ private boolean haveConditionsAfterNode(String parentNodeName){
- List postTaskList = new ArrayList<>();
+ boolean result = false;
Collection startVertex = DagHelper.getStartVertex(parentNodeName, dag, completeTaskList);
if(startVertex == null){
- return postTaskList;
+ return result;
+ }
+ for(String nodeName : startVertex){
+ TaskNode taskNode = dag.getNode(nodeName);
+ if(taskNode.getType().equals(TaskType.CONDITIONS.toString())){
+ result = true;
+ break;
+ }
}
+ return result;
+ }
- for (String nodeName : startVertex){
- // encapsulation task instance
- TaskInstance taskInstance = createTaskInstance(processInstance, nodeName ,
- dag.getNode(nodeName),parentNodeName);
- postTaskList.add(taskInstance);
+ /**
+ * if all of the task dependence are skip, skip it too.
+ * @param taskNode
+ * @return
+ */
+ private boolean isTaskNodeNeedSkip(TaskNode taskNode){
+ if(CollectionUtils.isEmpty(taskNode.getDepList())){
+ return false;
}
- return postTaskList;
+ for(String depNode : taskNode.getDepList()){
+ if(!skipTaskNodeList.containsKey(depNode)){
+ return false;
+ }
+ }
+ return true;
}
/**
- * return start task node list
- * @return task instance list
+ * set task node skip if dependence all skip
+ * @param taskNodesSkipList
*/
- private List getStartSubmitTaskList(){
+ private void setTaskNodeSkip(List taskNodesSkipList){
+ for(String skipNode : taskNodesSkipList){
+ skipTaskNodeList.putIfAbsent(skipNode, dag.getNode(skipNode));
+ Collection postNodeList = DagHelper.getStartVertex(skipNode, dag, completeTaskList);
+ List postSkipList = new ArrayList<>();
+ for(String post : postNodeList){
+ TaskNode postNode = dag.getNode(post);
+ if(isTaskNodeNeedSkip(postNode)){
+ postSkipList.add(post);
+ }
+ }
+ setTaskNodeSkip(postSkipList);
+ }
+ }
- List startTaskList = getPostTaskInstanceByNode(dag, null);
- HashMap successTaskMaps = new HashMap<>();
- List resultList = new ArrayList<>();
- while(Stopper.isRunning()){
- for(TaskInstance task : startTaskList){
- if(task.getState().typeIsSuccess()){
- successTaskMaps.put(task.getName(), task);
- }else if(!completeTaskList.containsKey(task.getName()) && !errorTaskList.containsKey(task.getName())){
- resultList.add(task);
+ /**
+ * parse condition task find the branch process
+ * set skip flag for another one.
+ * @param nodeName
+ * @return
+ */
+ private List parseConditionTask(String nodeName){
+ List conditionTaskList = new ArrayList<>();
+ TaskNode taskNode = dag.getNode(nodeName);
+ if(!taskNode.isConditionsTask()){
+ return conditionTaskList;
+ }
+ ConditionsParameters conditionsParameters =
+ JSONUtils.parseObject(taskNode.getConditionResult(), ConditionsParameters.class);
+
+ TaskInstance taskInstance = completeTaskList.get(nodeName);
+ if(taskInstance == null){
+ logger.error("task instance cannot find, please check it!", nodeName);
+ return conditionTaskList;
+ }
+
+ if(taskInstance.getState().typeIsSuccess()){
+ conditionTaskList = conditionsParameters.getSuccessNode();
+ setTaskNodeSkip(conditionsParameters.getFailedNode());
+ }else if(taskInstance.getState().typeIsFailure()){
+ conditionTaskList = conditionsParameters.getFailedNode();
+ setTaskNodeSkip(conditionsParameters.getSuccessNode());
+ }else{
+ conditionTaskList.add(nodeName);
+ }
+ return conditionTaskList;
+ }
+
+ /**
+ * parse post node list of previous node
+ * if condition node: return process according to the settings
+ * if post node completed, return post nodes of the completed node
+ * @param previousNodeName
+ * @return
+ */
+ private List parsePostNodeList(String previousNodeName){
+ List postNodeList = new ArrayList<>();
+
+ TaskNode taskNode = dag.getNode(previousNodeName);
+ if(taskNode != null && taskNode.isConditionsTask()){
+ return parseConditionTask(previousNodeName);
+ }
+ Collection postNodeCollection = DagHelper.getStartVertex(previousNodeName, dag, completeTaskList);
+ List postSkipList = new ArrayList<>();
+ // delete success node, parse the past nodes
+ // if conditions node,
+ // 1. parse the branch process according the conditions setting
+ // 2. set skip flag on anther branch process
+ for(String postNode : postNodeCollection){
+ if(completeTaskList.containsKey(postNode)){
+ TaskInstance postTaskInstance = completeTaskList.get(postNode);
+ if(dag.getNode(postNode).isConditionsTask()){
+ List conditionTaskNodeList = parseConditionTask(postNode);
+ for(String conditions : conditionTaskNodeList){
+ postNodeList.addAll(parsePostNodeList(conditions));
+ }
+ }else if(postTaskInstance.getState().typeIsSuccess()){
+ postNodeList.addAll(parsePostNodeList(postNode));
+ }else{
+ postNodeList.add(postNode);
}
- }
- startTaskList.clear();
- if(successTaskMaps.size() == 0){
- break;
- }
- Set taskNameKeys = successTaskMaps.keySet();
- for(String taskName : taskNameKeys){
- startTaskList.addAll(getPostTaskInstanceByNode(dag, taskName));
+ }else if(isTaskNodeNeedSkip(dag.getNode(postNode))){
+ postSkipList.add(postNode);
+ setTaskNodeSkip(postSkipList);
+ postSkipList.clear();
+ }else{
+ postNodeList.add(postNode);
}
- successTaskMaps.clear();
}
- return resultList;
+ return postNodeList;
}
/**
@@ -543,14 +632,17 @@ public class MasterExecThread implements Runnable {
*/
private void submitPostNode(String parentNodeName){
- List submitTaskList = null;
- if(parentNodeName == null){
- submitTaskList = getStartSubmitTaskList();
- }else{
- submitTaskList = getPostTaskInstanceByNode(dag, parentNodeName);
+ List submitTaskNodeList = parsePostNodeList(parentNodeName);
+
+ List taskInstances = new ArrayList<>();
+ for(String taskNode : submitTaskNodeList){
+ taskInstances.add(createTaskInstance(processInstance, taskNode,
+ dag.getNode(taskNode)));
}
+
// if previous node success , post node submit
- for(TaskInstance task : submitTaskList){
+ for(TaskInstance task : taskInstances){
+
if(readyToSubmitTaskList.containsKey(task.getName())){
continue;
}
@@ -574,27 +666,31 @@ public class MasterExecThread implements Runnable {
private DependResult isTaskDepsComplete(String taskName) {
Collection startNodes = dag.getBeginNode();
- // if the vertex returns true directly
+ // if vertex,returns true directly
if(startNodes.contains(taskName)){
return DependResult.SUCCESS;
}
TaskNode taskNode = dag.getNode(taskName);
- List depsNameList = taskNode.getDepList();
- for(String depsNode : depsNameList ){
+ List depNameList = taskNode.getDepList();
+ for(String depsNode : depNameList ){
- if(forbiddenTaskList.containsKey(depsNode)){
+ if(forbiddenTaskList.containsKey(depsNode) ||
+ skipTaskNodeList.containsKey(depsNode)){
continue;
}
// dependencies must be fully completed
if(!completeTaskList.containsKey(depsNode)){
return DependResult.WAITING;
}
- ExecutionStatus taskState = completeTaskList.get(depsNode).getState();
- if(taskState.typeIsFailure()){
- return DependResult.FAILED;
+ ExecutionStatus depTaskState = completeTaskList.get(depsNode).getState();
+ // conditions task would not return failed.
+ if(depTaskState.typeIsFailure()){
+ if(!haveConditionsAfterNode(depsNode) && !dag.getNode(depsNode).isConditionsTask()){
+ return DependResult.FAILED;
+ }
}
- if(taskState.typeIsPause() || taskState.typeIsCancel()){
+ if(depTaskState.typeIsPause() || depTaskState.typeIsCancel()){
return DependResult.WAITING;
}
}
@@ -706,7 +802,7 @@ public class MasterExecThread implements Runnable {
* @return process instance execution status
*/
private ExecutionStatus getProcessInstanceState(){
- ProcessInstance instance = processDao.findProcessInstanceById(processInstance.getId());
+ ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId());
ExecutionStatus state = instance.getState();
if(activeTaskNode.size() > 0){
@@ -784,10 +880,10 @@ public class MasterExecThread implements Runnable {
processInstance.getState().toString(), state.toString(),
processInstance.getCommandType().toString());
processInstance.setState(state);
- ProcessInstance instance = processDao.findProcessInstanceById(processInstance.getId());
+ ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId());
instance.setState(state);
instance.setProcessDefinition(processInstance.getProcessDefinition());
- processDao.updateProcessInstance(instance);
+ processService.updateProcessInstance(instance);
processInstance = instance;
}
}
@@ -845,7 +941,7 @@ public class MasterExecThread implements Runnable {
// send warning email if process time out.
if( !sendTimeWarning && checkProcessTimeOut(processInstance) ){
alertManager.sendProcessTimeoutAlert(processInstance,
- processDao.findProcessDefineById(processInstance.getProcessDefinitionId()));
+ processService.findProcessDefineById(processInstance.getProcessDefinitionId()));
sendTimeWarning = true;
}
for(Map.Entry> entry: activeTaskNode.entrySet()) {
@@ -877,11 +973,15 @@ public class MasterExecThread implements Runnable {
if(task.taskCanRetry()){
addTaskToStandByList(task);
}else{
- // node failure, based on failure strategy
- errorTaskList.put(task.getName(), task);
completeTaskList.put(task.getName(), task);
- if(processInstance.getFailureStrategy() == FailureStrategy.END){
- killTheOtherTasks();
+ if( task.getTaskType().equals(TaskType.CONDITIONS.toString()) ||
+ haveConditionsAfterNode(task.getName())) {
+ submitPostNode(task.getName());
+ }else{
+ errorTaskList.put(task.getName(), task);
+ if(processInstance.getFailureStrategy() == FailureStrategy.END){
+ killTheOtherTasks();
+ }
}
}
continue;
@@ -903,7 +1003,7 @@ public class MasterExecThread implements Runnable {
if(completeTask.getState()== ExecutionStatus.PAUSE){
completeTask.setState(ExecutionStatus.KILL);
completeTaskList.put(entry.getKey(), completeTask);
- processDao.updateTaskInstance(completeTask);
+ processService.updateTaskInstance(completeTask);
}
}
}
@@ -961,7 +1061,7 @@ public class MasterExecThread implements Runnable {
Future future = entry.getValue();
TaskInstance taskInstance = taskExecThread.getTaskInstance();
- taskInstance = processDao.findTaskInstanceById(taskInstance.getId());
+ taskInstance = processService.findTaskInstanceById(taskInstance.getId());
if(taskInstance.getState().typeIsFinished()){
continue;
}
@@ -1031,7 +1131,7 @@ public class MasterExecThread implements Runnable {
}
try {
Integer intId = Integer.valueOf(taskId);
- TaskInstance task = processDao.findTaskInstanceById(intId);
+ TaskInstance task = processService.findTaskInstanceById(intId);
if(task == null){
logger.error("start node id cannot be found: {}", taskId);
}else {
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java
index a873fb786d..cc5a7e76e4 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java
@@ -22,13 +22,13 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
-import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.zk.ZKMasterClient;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.service.zk.AbstractZKClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,7 +53,7 @@ public class MasterSchedulerThread implements Runnable {
/**
* dolphinscheduler database interface
*/
- private final ProcessDao processDao;
+ private final ProcessService processService;
/**
* zookeeper master client
@@ -74,11 +74,11 @@ public class MasterSchedulerThread implements Runnable {
/**
* constructor of MasterSchedulerThread
* @param zkClient zookeeper master client
- * @param processDao process dao
+ * @param processService process service
* @param masterExecThreadNum master exec thread num
*/
- public MasterSchedulerThread(ZKMasterClient zkClient, ProcessDao processDao, int masterExecThreadNum){
- this.processDao = processDao;
+ public MasterSchedulerThread(ZKMasterClient zkClient, ProcessService processService, int masterExecThreadNum){
+ this.processService = processService;
this.zkMasterClient = zkClient;
this.masterExecThreadNum = masterExecThreadNum;
this.masterExecService = ThreadUtils.newDaemonFixedThreadExecutor("Master-Exec-Thread",masterExecThreadNum);
@@ -115,19 +115,19 @@ public class MasterSchedulerThread implements Runnable {
ThreadPoolExecutor poolExecutor = (ThreadPoolExecutor) masterExecService;
int activeCount = poolExecutor.getActiveCount();
// make sure to scan and delete command table in one transaction
- Command command = processDao.findOneCommand();
+ Command command = processService.findOneCommand();
if (command != null) {
- logger.info(String.format("find one command: id: %d, type: %s", command.getId(),command.getCommandType().toString()));
+ logger.info("find one command: id: {}, type: {}", command.getId(),command.getCommandType());
try{
- processInstance = processDao.handleCommand(logger, OSUtils.getHost(), this.masterExecThreadNum - activeCount, command);
+ processInstance = processService.handleCommand(logger, OSUtils.getHost(), this.masterExecThreadNum - activeCount, command);
if (processInstance != null) {
logger.info("start master exec thread , split DAG ...");
- masterExecService.execute(new MasterExecThread(processInstance,processDao));
+ masterExecService.execute(new MasterExecThread(processInstance, processService));
}
}catch (Exception e){
logger.error("scan command error ", e);
- processDao.moveToErrorCommand(command, e.toString());
+ processService.moveToErrorCommand(command, e.toString());
}
} else{
//indicate that no command ,sleep for 1s
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java
index f2ee66b64a..66d1a3f4c2 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java
@@ -82,7 +82,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread {
result = waitTaskQuit();
}
taskInstance.setEndTime(new Date());
- processDao.updateTaskInstance(taskInstance);
+ processService.updateTaskInstance(taskInstance);
logger.info("task :{} id:{}, process id:{}, exec thread completed ",
this.taskInstance.getName(),taskInstance.getId(), processInstance.getId() );
return result;
@@ -94,7 +94,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread {
*/
public Boolean waitTaskQuit(){
// query new state
- taskInstance = processDao.findTaskInstanceById(taskInstance.getId());
+ taskInstance = processService.findTaskInstanceById(taskInstance.getId());
logger.info("wait task: process id: {}, task id:{}, task name:{} complete",
this.taskInstance.getProcessInstanceId(), this.taskInstance.getId(), this.taskInstance.getName());
// task time out
@@ -126,15 +126,15 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread {
if (remainTime < 0) {
logger.warn("task id: {} execution time out",taskInstance.getId());
// process define
- ProcessDefinition processDefine = processDao.findProcessDefineById(processInstance.getProcessDefinitionId());
+ ProcessDefinition processDefine = processService.findProcessDefineById(processInstance.getProcessDefinitionId());
// send warn mail
alertDao.sendTaskTimeoutAlert(processInstance.getWarningGroupId(),processDefine.getReceivers(),processDefine.getReceiversCc(),taskInstance.getId(),taskInstance.getName());
checkTimeout = false;
}
}
// updateProcessInstance task instance
- taskInstance = processDao.findTaskInstanceById(taskInstance.getId());
- processInstance = processDao.findProcessInstanceById(processInstance.getId());
+ taskInstance = processService.findTaskInstanceById(taskInstance.getId());
+ processInstance = processService.findProcessInstanceById(processInstance.getId());
Thread.sleep(Constants.SLEEP_TIME_MILLIS);
} catch (Exception e) {
logger.error("exception",e);
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java
index 0026de7c25..fc16b5112b 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java
@@ -64,7 +64,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread {
}
setTaskInstanceState();
waitTaskQuit();
- subProcessInstance = processDao.findSubProcessInstance(processInstance.getId(), taskInstance.getId());
+ subProcessInstance = processService.findSubProcessInstance(processInstance.getId(), taskInstance.getId());
// at the end of the subflow , the task state is changed to the subflow state
if(subProcessInstance != null){
@@ -75,7 +75,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread {
}
}
taskInstance.setEndTime(new Date());
- processDao.updateTaskInstance(taskInstance);
+ processService.updateTaskInstance(taskInstance);
logger.info("subflow task :{} id:{}, process id:{}, exec thread completed ",
this.taskInstance.getName(),taskInstance.getId(), processInstance.getId() );
result = true;
@@ -96,14 +96,14 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread {
* @return
*/
private Boolean setTaskInstanceState(){
- subProcessInstance = processDao.findSubProcessInstance(processInstance.getId(), taskInstance.getId());
+ subProcessInstance = processService.findSubProcessInstance(processInstance.getId(), taskInstance.getId());
if(subProcessInstance == null || taskInstance.getState().typeIsFinished()){
return false;
}
taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
taskInstance.setStartTime(new Date());
- processDao.updateTaskInstance(taskInstance);
+ processService.updateTaskInstance(taskInstance);
return true;
}
@@ -111,7 +111,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread {
* updateProcessInstance parent state
*/
private void updateParentProcessState(){
- ProcessInstance parentProcessInstance = processDao.findProcessInstanceById(this.processInstance.getId());
+ ProcessInstance parentProcessInstance = processService.findProcessInstanceById(this.processInstance.getId());
if(parentProcessInstance == null){
logger.error("parent work flow instance is null , please check it! work flow id {}", processInstance.getId());
@@ -145,7 +145,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread {
continue;
}
}
- subProcessInstance = processDao.findProcessInstanceById(subProcessInstance.getId());
+ subProcessInstance = processService.findProcessInstanceById(subProcessInstance.getId());
updateParentProcessState();
if (subProcessInstance.getState().typeIsFinished()){
break;
@@ -171,7 +171,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread {
return;
}
subProcessInstance.setState(ExecutionStatus.READY_STOP);
- processDao.updateProcessInstance(subProcessInstance);
+ processService.updateProcessInstance(subProcessInstance);
}
/**
@@ -183,6 +183,6 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread {
return;
}
subProcessInstance.setState(ExecutionStatus.READY_PAUSE);
- processDao.updateProcessInstance(subProcessInstance);
+ processService.updateProcessInstance(subProcessInstance);
}
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/ZKMonitorImpl.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/ZKMonitorImpl.java
index 927074012d..5acc8fd931 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/ZKMonitorImpl.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/ZKMonitorImpl.java
@@ -16,7 +16,7 @@
*/
package org.apache.dolphinscheduler.server.monitor;
-import org.apache.dolphinscheduler.common.zk.ZookeeperOperator;
+import org.apache.dolphinscheduler.service.zk.ZookeeperOperator;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java
deleted file mode 100644
index 1c6c97b88f..0000000000
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LogClient.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.server.rpc;
-
-import io.grpc.ManagedChannel;
-import io.grpc.ManagedChannelBuilder;
-import io.grpc.StatusRuntimeException;
-import org.apache.dolphinscheduler.rpc.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * log client
- */
-public class LogClient {
-
- /**
- * logger of LogClient
- */
- private static final Logger logger = LoggerFactory.getLogger(LogClient.class);
-
- /**
- * managed channel
- */
- private final ManagedChannel channel;
-
- /**
- * blocking stub
- */
- private final LogViewServiceGrpc.LogViewServiceBlockingStub blockingStub;
-
- /**
- * Construct client connecting to HelloWorld server at host:port.
- *
- * @param host host
- * @param port port
- */
- public LogClient(String host, int port) {
- this(ManagedChannelBuilder.forAddress(host, port)
- // Channels are secure by default (via SSL/TLS). For the example we disable TLS to avoid
- // needing certificates.
- .usePlaintext(true));
- }
-
- /**
- * Construct client for accessing RouteGuide server using the existing channel.
- *
- * @param channelBuilder channel builder
- */
- LogClient(ManagedChannelBuilder channelBuilder) {
- /**
- * set max message read size
- */
- channelBuilder.maxInboundMessageSize(Integer.MAX_VALUE);
- channel = channelBuilder.build();
- blockingStub = LogViewServiceGrpc.newBlockingStub(channel);
- }
-
- /**
- * shut down channel
- *
- * @throws InterruptedException interrupted exception
- */
- public void shutdown() throws InterruptedException {
- channel.shutdown().awaitTermination(5, TimeUnit.SECONDS);
- }
-
- /**
- * roll view log
- *
- * @param path log path
- * @param skipLineNum skip line num
- * @param limit limit
- * @return log content
- */
- public String rollViewLog(String path,int skipLineNum,int limit) {
- logger.info("roll view log , path : {},skipLineNum : {} ,limit :{}", path, skipLineNum, limit);
- LogParameter pathParameter = LogParameter
- .newBuilder()
- .setPath(path)
- .setSkipLineNum(skipLineNum)
- .setLimit(limit)
- .build();
- RetStrInfo retStrInfo;
- try {
- retStrInfo = blockingStub.rollViewLog(pathParameter);
- return retStrInfo.getMsg();
- } catch (StatusRuntimeException e) {
- logger.error("roll view log failed", e);
- return null;
- }
- }
-
- /**
- * view all log
- *
- * @param path log path
- * @return log content
- */
- public String viewLog(String path) {
- logger.info("view log path : {}",path);
-
- PathParameter pathParameter = PathParameter.newBuilder().setPath(path).build();
- RetStrInfo retStrInfo;
- try {
- retStrInfo = blockingStub.viewLog(pathParameter);
- return retStrInfo.getMsg();
- } catch (StatusRuntimeException e) {
- logger.error("view log failed", e);
- return null;
- }
- }
-
- /**
- * get log bytes
- *
- * @param path log path
- * @return log content
- */
- public byte[] getLogBytes(String path) {
- logger.info("get log bytes {}",path);
-
- PathParameter pathParameter = PathParameter.newBuilder().setPath(path).build();
- RetByteInfo retByteInfo;
- try {
- retByteInfo = blockingStub.getLogBytes(pathParameter);
- return retByteInfo.getData().toByteArray();
- } catch (StatusRuntimeException e) {
- logger.error("get log bytes failed ", e);
- return null;
- }
- }
-}
\ No newline at end of file
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LoggerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LoggerServer.java
deleted file mode 100644
index 5ec5df92fc..0000000000
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/rpc/LoggerServer.java
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.server.rpc;
-
-import io.grpc.stub.StreamObserver;
-import org.apache.dolphinscheduler.common.Constants;
-import com.google.protobuf.ByteString;
-import io.grpc.Server;
-import io.grpc.ServerBuilder;
-import org.apache.dolphinscheduler.rpc.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.*;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * logger server
- */
-public class LoggerServer {
-
- private static final Logger logger = LoggerFactory.getLogger(LoggerServer.class);
-
- /**
- * server
- */
- private Server server;
-
- /**
- * server start
- * @throws IOException io exception
- */
- public void start() throws IOException {
- /* The port on which the server should run */
- int port = Constants.RPC_PORT;
- server = ServerBuilder.forPort(port)
- .addService(new LogViewServiceGrpcImpl())
- .build()
- .start();
- logger.info("server started, listening on port : {}" , port);
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- // Use stderr here since the logger may have been reset by its JVM shutdown hook.
- logger.info("shutting down gRPC server since JVM is shutting down");
- LoggerServer.this.stop();
- logger.info("server shut down");
- }
- });
- }
-
- /**
- * stop
- */
- private void stop() {
- if (server != null) {
- server.shutdown();
- }
- }
-
- /**
- * await termination on the main thread since the grpc library uses daemon threads.
- */
- private void blockUntilShutdown() throws InterruptedException {
- if (server != null) {
- server.awaitTermination();
- }
- }
-
- /**
- * main launches the server from the command line.
- */
-
- /**
- * main launches the server from the command line.
- * @param args arguments
- * @throws IOException io exception
- * @throws InterruptedException interrupted exception
- */
- public static void main(String[] args) throws IOException, InterruptedException {
- final LoggerServer server = new LoggerServer();
- server.start();
- server.blockUntilShutdown();
- }
-
- /**
- * Log View Service Grpc Implementation
- */
- static class LogViewServiceGrpcImpl extends LogViewServiceGrpc.LogViewServiceImplBase {
- @Override
- public void rollViewLog(LogParameter request, StreamObserver responseObserver) {
-
- logger.info("log parameter path : {} ,skip line : {}, limit : {}",
- request.getPath(),
- request.getSkipLineNum(),
- request.getLimit());
- List list = readFile(request.getPath(), request.getSkipLineNum(), request.getLimit());
- StringBuilder sb = new StringBuilder();
- boolean errorLineFlag = false;
- for (String line : list){
- sb.append(line + "\r\n");
- }
- RetStrInfo retInfoBuild = RetStrInfo.newBuilder().setMsg(sb.toString()).build();
- responseObserver.onNext(retInfoBuild);
- responseObserver.onCompleted();
- }
-
- @Override
- public void viewLog(PathParameter request, StreamObserver responseObserver) {
- logger.info("task path is : {} " , request.getPath());
- RetStrInfo retInfoBuild = RetStrInfo.newBuilder().setMsg(readFile(request.getPath())).build();
- responseObserver.onNext(retInfoBuild);
- responseObserver.onCompleted();
- }
-
- @Override
- public void getLogBytes(PathParameter request, StreamObserver responseObserver) {
- try {
- ByteString bytes = ByteString.copyFrom(getFileBytes(request.getPath()));
- RetByteInfo.Builder builder = RetByteInfo.newBuilder();
- builder.setData(bytes);
- responseObserver.onNext(builder.build());
- responseObserver.onCompleted();
- }catch (Exception e){
- logger.error("get log bytes failed",e);
- }
- }
- }
-
- /**
- * get files bytes
- *
- * @param path path
- * @return byte array of file
- * @throws Exception exception
- */
- private static byte[] getFileBytes(String path){
- InputStream in = null;
- ByteArrayOutputStream bos = null;
- try {
- in = new FileInputStream(path);
- bos = new ByteArrayOutputStream();
- byte[] buf = new byte[1024];
- int len = 0;
- while ((len = in.read(buf)) != -1) {
- bos.write(buf, 0, len);
- }
- return bos.toByteArray();
- }catch (IOException e){
- logger.error("get file bytes error",e);
- }finally {
- if (bos != null){
- try {
- bos.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- if (in != null){
- try {
- in.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- return null;
- }
-
- /**
- * read file content
- *
- * @param path
- * @param skipLine
- * @param limit
- * @return
- */
- private static List readFile(String path,int skipLine,int limit){
- try (Stream stream = Files.lines(Paths.get(path))) {
- return stream.skip(skipLine).limit(limit).collect(Collectors.toList());
- } catch (IOException e) {
- logger.error("read file failed",e);
- }
- return null;
- }
-
- /**
- * read file content
- *
- * @param path path
- * @return string of file content
- * @throws Exception exception
- */
- private static String readFile(String path){
- BufferedReader br = null;
- String line = null;
- StringBuilder sb = new StringBuilder();
- try {
- br = new BufferedReader(new InputStreamReader(new FileInputStream(path)));
- boolean errorLineFlag = false;
- while ((line = br.readLine()) != null){
- sb.append(line + "\r\n");
- }
-
- return sb.toString();
- }catch (IOException e){
- logger.error("read file failed",e);
- }finally {
- try {
- if (br != null){
- br.close();
- }
- } catch (IOException e) {
- logger.error(e.getMessage(),e);
- }
- }
- return null;
- }
-
-}
\ No newline at end of file
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java
index ab34ddfc2b..4c33ef8db2 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java
@@ -87,7 +87,7 @@ public class FlinkArgsUtils {
args.add(taskManagerMemory);
}
- args.add(Constants.FLINK_detach); //-d
+ args.add(Constants.FLINK_DETACH); //-d
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
index fd0a08cd8e..e0c00c55d9 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
@@ -22,8 +22,8 @@ import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.server.rpc.LogClient;
import org.apache.commons.io.FileUtils;
+import org.apache.dolphinscheduler.service.log.LogClientService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -375,9 +375,16 @@ public class ProcessUtils {
public static void killYarnJob(TaskInstance taskInstance) {
try {
Thread.sleep(Constants.SLEEP_TIME_MILLIS);
- LogClient logClient = new LogClient(taskInstance.getHost(), Constants.RPC_PORT);
-
- String log = logClient.viewLog(taskInstance.getLogPath());
+ LogClientService logClient = null;
+ String log = null;
+ try {
+ logClient = new LogClientService();
+ log = logClient.viewLog(taskInstance.getHost(), Constants.RPC_PORT, taskInstance.getLogPath());
+ } finally {
+ if(logClient != null){
+ logClient.close();
+ }
+ }
if (StringUtils.isNotEmpty(log)) {
List appIds = LoggerUtils.getAppIds(log, logger);
String workerDir = taskInstance.getExecutePath();
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java
index 7264c2f59d..5550e750b5 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/RemoveZKNode.java
@@ -16,7 +16,7 @@
*/
package org.apache.dolphinscheduler.server.utils;
-import org.apache.dolphinscheduler.common.zk.ZookeeperOperator;
+import org.apache.dolphinscheduler.service.zk.ZookeeperOperator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
index d270880408..ace93079ff 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
@@ -22,22 +22,22 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.IStoppable;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.TaskType;
-import org.apache.dolphinscheduler.common.queue.ITaskQueue;
-import org.apache.dolphinscheduler.common.queue.TaskQueueFactory;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
-import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
import org.apache.dolphinscheduler.dao.AlertDao;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.ProcessUtils;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.server.worker.runner.FetchTaskThread;
import org.apache.dolphinscheduler.server.zk.ZKWorkerClient;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.service.queue.ITaskQueue;
+import org.apache.dolphinscheduler.service.queue.TaskQueueFactory;
+import org.apache.dolphinscheduler.service.zk.AbstractZKClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -73,10 +73,10 @@ public class WorkerServer implements IStoppable {
/**
- * process database access
+ * process service
*/
@Autowired
- private ProcessDao processDao;
+ private ProcessService processService;
/**
* alert database access
@@ -104,13 +104,6 @@ public class WorkerServer implements IStoppable {
*/
private ExecutorService fetchTaskExecutorService;
- /**
- * spring application context
- * only use it for initialization
- */
- @Autowired
- private SpringApplicationContext springApplicationContext;
-
/**
* CountDownLatch latch
*/
@@ -122,6 +115,13 @@ public class WorkerServer implements IStoppable {
@Autowired
private WorkerConfig workerConfig;
+ /**
+ * spring application context
+ * only use it for initialization
+ */
+ @Autowired
+ private SpringApplicationContext springApplicationContext;
+
/**
* master server startup
*
@@ -149,7 +149,7 @@ public class WorkerServer implements IStoppable {
this.fetchTaskExecutorService = ThreadUtils.newDaemonSingleThreadExecutor("Worker-Fetch-Thread-Executor");
- heartbeatWorkerService = ThreadUtils.newDaemonThreadScheduledExecutor("Worker-Heartbeat-Thread-Executor", Constants.defaulWorkerHeartbeatThreadNum);
+ heartbeatWorkerService = ThreadUtils.newDaemonThreadScheduledExecutor("Worker-Heartbeat-Thread-Executor", Constants.DEFAUL_WORKER_HEARTBEAT_THREAD_NUM);
// heartbeat thread implement
Runnable heartBeatThread = heartBeatThread();
@@ -167,7 +167,7 @@ public class WorkerServer implements IStoppable {
killExecutorService.execute(killProcessThread);
// new fetch task thread
- FetchTaskThread fetchTaskThread = new FetchTaskThread(zkWorkerClient, processDao, taskQueue);
+ FetchTaskThread fetchTaskThread = new FetchTaskThread(zkWorkerClient, processService, taskQueue);
// submit fetch task thread
fetchTaskExecutorService.execute(fetchTaskThread);
@@ -297,7 +297,7 @@ public class WorkerServer implements IStoppable {
Set taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL);
if (CollectionUtils.isNotEmpty(taskInfoSet)){
for (String taskInfo : taskInfoSet){
- killTask(taskInfo, processDao);
+ killTask(taskInfo, processService);
removeKillInfoFromQueue(taskInfo);
}
}
@@ -319,7 +319,7 @@ public class WorkerServer implements IStoppable {
* @param taskInfo task info
* @param pd process dao
*/
- private void killTask(String taskInfo, ProcessDao pd) {
+ private void killTask(String taskInfo, ProcessService pd) {
logger.info("get one kill command from tasks kill queue: " + taskInfo);
String[] taskInfoArray = taskInfo.split("-");
if(taskInfoArray.length != 2){
@@ -357,7 +357,7 @@ public class WorkerServer implements IStoppable {
* @param taskInstance
* @param pd process dao
*/
- private void deleteTaskFromQueue(TaskInstance taskInstance, ProcessDao pd){
+ private void deleteTaskFromQueue(TaskInstance taskInstance, ProcessService pd){
// creating distributed locks, lock path /dolphinscheduler/lock/worker
InterProcessMutex mutex = null;
logger.info("delete task from tasks queue: " + taskInstance.getId());
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java
index 221ad069bb..013db83761 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java
@@ -19,17 +19,18 @@ package org.apache.dolphinscheduler.server.worker.runner;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.common.queue.ITaskQueue;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.*;
-import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.server.zk.ZKWorkerClient;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.service.queue.ITaskQueue;
+import org.apache.dolphinscheduler.service.zk.AbstractZKClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,7 +64,7 @@ public class FetchTaskThread implements Runnable{
/**
* process database access
*/
- private final ProcessDao processDao;
+ private final ProcessService processService;
/**
* worker thread pool executor
@@ -91,10 +92,10 @@ public class FetchTaskThread implements Runnable{
private WorkerConfig workerConfig;
public FetchTaskThread(ZKWorkerClient zkWorkerClient,
- ProcessDao processDao,
+ ProcessService processService,
ITaskQueue taskQueue){
this.zkWorkerClient = zkWorkerClient;
- this.processDao = processDao;
+ this.processService = processService;
this.taskQueue = taskQueue;
this.workerConfig = SpringApplicationContext.getBean(WorkerConfig.class);
this.taskNum = workerConfig.getWorkerFetchTaskNum();
@@ -112,12 +113,12 @@ public class FetchTaskThread implements Runnable{
*/
private boolean checkWorkerGroup(TaskInstance taskInstance, String host){
- int taskWorkerGroupId = processDao.getTaskWorkerGroupId(taskInstance);
+ int taskWorkerGroupId = processService.getTaskWorkerGroupId(taskInstance);
if(taskWorkerGroupId <= 0){
return true;
}
- WorkerGroup workerGroup = processDao.queryWorkerGroupById(taskWorkerGroupId);
+ WorkerGroup workerGroup = processService.queryWorkerGroupById(taskWorkerGroupId);
if(workerGroup == null ){
logger.info("task {} cannot find the worker group, use all worker instead.", taskInstance.getId());
return true;
@@ -184,7 +185,7 @@ public class FetchTaskThread implements Runnable{
// mainly to wait for the master insert task to succeed
waitForTaskInstance();
- taskInstance = processDao.getTaskInstanceDetailByTaskId(taskInstId);
+ taskInstance = processService.getTaskInstanceDetailByTaskId(taskInstId);
// verify task instance is null
if (verifyTaskInstanceIsNull(taskInstance)) {
@@ -200,7 +201,7 @@ public class FetchTaskThread implements Runnable{
// if process definition is null ,process definition already deleted
int userId = taskInstance.getProcessDefine() == null ? 0 : taskInstance.getProcessDefine().getUserId();
- Tenant tenant = processDao.getTenantForProcess(
+ Tenant tenant = processService.getTenantForProcess(
taskInstance.getProcessInstance().getTenantId(),
userId);
@@ -212,7 +213,7 @@ public class FetchTaskThread implements Runnable{
}
// set queue for process instance, user-specified queue takes precedence over tenant queue
- String userQueue = processDao.queryUserQueueByProcessInstanceId(taskInstance.getProcessInstanceId());
+ String userQueue = processService.queryUserQueueByProcessInstanceId(taskInstance.getProcessInstanceId());
taskInstance.getProcessInstance().setQueue(StringUtils.isEmpty(userQueue) ? tenant.getQueue() : userQueue);
taskInstance.getProcessInstance().setTenantCode(tenant.getTenantCode());
@@ -234,7 +235,7 @@ public class FetchTaskThread implements Runnable{
logger.info("task : {} ready to submit to task scheduler thread",taskInstId);
// submit task
- workerExecService.submit(new TaskScheduleThread(taskInstance, processDao));
+ workerExecService.submit(new TaskScheduleThread(taskInstance, processService));
// remove node from zk
removeNodeFromTaskQueue(taskQueueStr);
@@ -259,7 +260,7 @@ public class FetchTaskThread implements Runnable{
removeNodeFromTaskQueue(taskQueueStr);
if (taskInstance != null){
- processDao.changeTaskState(ExecutionStatus.FAILURE,
+ processService.changeTaskState(ExecutionStatus.FAILURE,
taskInstance.getStartTime(),
taskInstance.getHost(),
null,
@@ -347,7 +348,7 @@ public class FetchTaskThread implements Runnable{
int retryTimes = 30;
while (taskInstance == null && retryTimes > 0) {
Thread.sleep(Constants.SLEEP_TIME_MILLIS);
- taskInstance = processDao.findTaskInstanceById(taskInstId);
+ taskInstance = processService.findTaskInstanceById(taskInstId);
retryTimes--;
}
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java
index f179d6344a..5e68acf94e 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java
@@ -31,15 +31,15 @@ import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.dao.permission.PermissionCheck;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.log.TaskLogDiscriminator;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskManager;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.permission.PermissionCheck;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,9 +64,9 @@ public class TaskScheduleThread implements Runnable {
private TaskInstance taskInstance;
/**
- * process database access
+ * process service
*/
- private final ProcessDao processDao;
+ private final ProcessService processService;
/**
* abstract task
@@ -77,10 +77,10 @@ public class TaskScheduleThread implements Runnable {
* constructor
*
* @param taskInstance task instance
- * @param processDao process dao
+ * @param processService process dao
*/
- public TaskScheduleThread(TaskInstance taskInstance, ProcessDao processDao){
- this.processDao = processDao;
+ public TaskScheduleThread(TaskInstance taskInstance, ProcessService processService){
+ this.processService = processService;
this.taskInstance = taskInstance;
}
@@ -152,7 +152,7 @@ public class TaskScheduleThread implements Runnable {
logger.error("task scheduler failure", e);
kill();
// update task instance state
- processDao.changeTaskState(ExecutionStatus.FAILURE,
+ processService.changeTaskState(ExecutionStatus.FAILURE,
new Date(),
taskInstance.getId());
}
@@ -161,11 +161,10 @@ public class TaskScheduleThread implements Runnable {
taskInstance.getId(),
task.getExitStatus());
// update task instance state
- processDao.changeTaskState(task.getExitStatus(),
+ processService.changeTaskState(task.getExitStatus(),
new Date(),
taskInstance.getId());
}
-
/**
* get global paras map
* @return
@@ -191,14 +190,14 @@ public class TaskScheduleThread implements Runnable {
// update task status is running
if(taskType.equals(TaskType.SQL.name()) ||
taskType.equals(TaskType.PROCEDURE.name())){
- processDao.changeTaskState(ExecutionStatus.RUNNING_EXEUTION,
+ processService.changeTaskState(ExecutionStatus.RUNNING_EXEUTION,
taskInstance.getStartTime(),
taskInstance.getHost(),
null,
getTaskLogPath(),
taskInstance.getId());
}else{
- processDao.changeTaskState(ExecutionStatus.RUNNING_EXEUTION,
+ processService.changeTaskState(ExecutionStatus.RUNNING_EXEUTION,
taskInstance.getStartTime(),
taskInstance.getHost(),
taskInstance.getExecutePath(),
@@ -212,21 +211,29 @@ public class TaskScheduleThread implements Runnable {
* @return log path
*/
private String getTaskLogPath() {
- String baseLog = ((TaskLogDiscriminator) ((SiftingAppender) ((LoggerContext) LoggerFactory.getILoggerFactory())
- .getLogger("ROOT")
- .getAppender("TASKLOGFILE"))
- .getDiscriminator()).getLogBase();
- if (baseLog.startsWith(Constants.SINGLE_SLASH)){
- return baseLog + Constants.SINGLE_SLASH +
- taskInstance.getProcessDefinitionId() + Constants.SINGLE_SLASH +
- taskInstance.getProcessInstanceId() + Constants.SINGLE_SLASH +
- taskInstance.getId() + ".log";
+ String logPath;
+ try{
+ String baseLog = ((TaskLogDiscriminator) ((SiftingAppender) ((LoggerContext) LoggerFactory.getILoggerFactory())
+ .getLogger("ROOT")
+ .getAppender("TASKLOGFILE"))
+ .getDiscriminator()).getLogBase();
+ if (baseLog.startsWith(Constants.SINGLE_SLASH)){
+ logPath = baseLog + Constants.SINGLE_SLASH +
+ taskInstance.getProcessDefinitionId() + Constants.SINGLE_SLASH +
+ taskInstance.getProcessInstanceId() + Constants.SINGLE_SLASH +
+ taskInstance.getId() + ".log";
+ }else{
+ logPath = System.getProperty("user.dir") + Constants.SINGLE_SLASH +
+ baseLog + Constants.SINGLE_SLASH +
+ taskInstance.getProcessDefinitionId() + Constants.SINGLE_SLASH +
+ taskInstance.getProcessInstanceId() + Constants.SINGLE_SLASH +
+ taskInstance.getId() + ".log";
+ }
+ }catch (Exception e){
+ logger.error("logger" + e);
+ logPath = "";
}
- return System.getProperty("user.dir") + Constants.SINGLE_SLASH +
- baseLog + Constants.SINGLE_SLASH +
- taskInstance.getProcessDefinitionId() + Constants.SINGLE_SLASH +
- taskInstance.getProcessInstanceId() + Constants.SINGLE_SLASH +
- taskInstance.getId() + ".log";
+ return logPath;
}
/**
@@ -311,7 +318,7 @@ public class TaskScheduleThread implements Runnable {
if (!resFile.exists()) {
try {
// query the tenant code of the resource according to the name of the resource
- String tentnCode = processDao.queryTenantCodeByResName(res);
+ String tentnCode = processService.queryTenantCodeByResName(res);
String resHdfsPath = HadoopUtils.getHdfsFilename(tentnCode, res);
logger.info("get resource file from hdfs :{}", resHdfsPath);
@@ -334,7 +341,7 @@ public class TaskScheduleThread implements Runnable {
private void checkDownloadPermission(List projectRes) throws Exception {
int userId = taskInstance.getProcessInstance().getExecutorId();
String[] resNames = projectRes.toArray(new String[projectRes.size()]);
- PermissionCheck permissionCheck = new PermissionCheck<>(AuthorizationType.RESOURCE_FILE,processDao,resNames,userId,logger);
+ PermissionCheck permissionCheck = new PermissionCheck<>(AuthorizationType.RESOURCE_FILE, processService,resNames,userId,logger);
permissionCheck.checkPermission();
}
}
\ No newline at end of file
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java
index 04098215dd..bac498c150 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java
@@ -21,10 +21,10 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.server.utils.ProcessUtils;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import java.io.*;
@@ -121,10 +121,10 @@ public abstract class AbstractCommandExecutor {
* task specific execution logic
*
* @param execCommand exec command
- * @param processDao process dao
+ * @param processService process dao
* @return exit status code
*/
- public int run(String execCommand, ProcessDao processDao) {
+ public int run(String execCommand, ProcessService processService) {
int exitStatusCode;
try {
@@ -147,7 +147,7 @@ public abstract class AbstractCommandExecutor {
// get process id
int pid = getProcessId(process);
- processDao.updatePidByTaskInstId(taskInstId, pid, "");
+ processService.updatePidByTaskInstId(taskInstId, pid, "");
logger.info("process start, process id is: {}", pid);
@@ -161,10 +161,10 @@ public abstract class AbstractCommandExecutor {
exitStatusCode = process.exitValue();
logger.info("process has exited, work dir:{}, pid:{} ,exitStatusCode:{}", taskDir, pid,exitStatusCode);
//update process state to db
- exitStatusCode = updateState(processDao, exitStatusCode, pid, taskInstId);
+ exitStatusCode = updateState(processService, exitStatusCode, pid, taskInstId);
} else {
- TaskInstance taskInstance = processDao.findTaskInstanceById(taskInstId);
+ TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId);
if (taskInstance == null) {
logger.error("task instance id:{} not exist", taskInstId);
} else {
@@ -176,7 +176,7 @@ public abstract class AbstractCommandExecutor {
} catch (InterruptedException e) {
exitStatusCode = -1;
- logger.error(String.format("interrupt exception: {}, task may be cancelled or killed",e.getMessage()), e);
+ logger.error("interrupt exception: {}, task may be cancelled or killed", e.getMessage(), e);
throw new RuntimeException("interrupt exception. exitCode is : " + exitStatusCode);
} catch (Exception e) {
exitStatusCode = -1;
@@ -219,23 +219,23 @@ public abstract class AbstractCommandExecutor {
/**
* update process state to db
*
- * @param processDao process dao
+ * @param processService process dao
* @param exitStatusCode exit status code
* @param pid process id
* @param taskInstId task instance id
* @return exit status code
*/
- private int updateState(ProcessDao processDao, int exitStatusCode, int pid, int taskInstId) {
+ private int updateState(ProcessService processService, int exitStatusCode, int pid, int taskInstId) {
//get yarn state by log
if (exitStatusCode == 0) {
- TaskInstance taskInstance = processDao.findTaskInstanceById(taskInstId);
+ TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId);
logger.info("process id is {}", pid);
List appIds = getAppLinks(taskInstance.getLogPath());
if (appIds.size() > 0) {
String appUrl = String.join(Constants.COMMA, appIds);
logger.info("yarn log url:{}",appUrl);
- processDao.updatePidByTaskInstId(taskInstId, pid, appUrl);
+ processService.updatePidByTaskInstId(taskInstId, pid, appUrl);
}
// check if all operations are completed
@@ -408,7 +408,7 @@ public abstract class AbstractCommandExecutor {
}
}
} catch (Exception e) {
- logger.error(String.format("yarn applications: %s status failed ", appIds.toString()),e);
+ logger.error("yarn applications: {} status failed ", appIds,e);
result = false;
}
return result;
@@ -458,7 +458,7 @@ public abstract class AbstractCommandExecutor {
lineList.add(line);
}
} catch (Exception e) {
- logger.error(String.format("read file: %s failed : ",filename),e);
+ logger.error("read file: {} failed",filename,e);
} finally {
if(br != null){
try {
@@ -535,7 +535,7 @@ public abstract class AbstractCommandExecutor {
/**
* when log buffer siz or flush time reach condition , then flush
*/
- if (logBuffer.size() >= Constants.defaultLogRowsNum || now - lastFlushTime > Constants.defaultLogFlushInterval) {
+ if (logBuffer.size() >= Constants.DEFAULT_LOG_ROWS_NUM || now - lastFlushTime > Constants.DEFAULT_LOG_FLUSH_INTERVAL) {
lastFlushTime = now;
/** log handle */
logHandler.accept(logBuffer);
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java
index f2772d0747..3795506b78 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java
@@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.enums.TaskRecordStatus;
import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
+import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters;
import org.apache.dolphinscheduler.common.task.datax.DataxParameters;
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
import org.apache.dolphinscheduler.common.task.mr.MapreduceParameters;
@@ -30,6 +31,7 @@ import org.apache.dolphinscheduler.common.task.python.PythonParameters;
import org.apache.dolphinscheduler.common.task.shell.ShellParameters;
import org.apache.dolphinscheduler.common.task.spark.SparkParameters;
import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.TaskRecordDao;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
@@ -198,6 +200,12 @@ public abstract class AbstractTask {
case DATAX:
paramsClass = DataxParameters.class;
break;
+ case SQOOP:
+ paramsClass = SqoopParameters.class;
+ break;
+ case CONDITIONS:
+ paramsClass = ConditionsParameters.class;
+ break;
default:
logger.error("not support this task type: {}", taskType);
throw new IllegalArgumentException("not support this task type");
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java
index 6846617408..39f4dfbb97 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java
@@ -16,10 +16,10 @@
*/
package org.apache.dolphinscheduler.server.worker.task;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.ProcessUtils;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
/**
@@ -39,7 +39,7 @@ public abstract class AbstractYarnTask extends AbstractTask {
/**
* process database access
*/
- protected ProcessDao processDao;
+ protected ProcessService processService;
/**
* Abstract Yarn Task
@@ -48,7 +48,7 @@ public abstract class AbstractYarnTask extends AbstractTask {
*/
public AbstractYarnTask(TaskProps taskProps, Logger logger) {
super(taskProps, logger);
- this.processDao = SpringApplicationContext.getBean(ProcessDao.class);
+ this.processService = SpringApplicationContext.getBean(ProcessService.class);
this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle,
taskProps.getTaskDir(),
taskProps.getTaskAppId(),
@@ -64,7 +64,7 @@ public abstract class AbstractYarnTask extends AbstractTask {
public void handle() throws Exception {
try {
// construct process
- exitStatusCode = shellCommandExecutor.run(buildCommand(), processDao);
+ exitStatusCode = shellCommandExecutor.run(buildCommand(), processService);
} catch (Exception e) {
logger.error("yarn process failure", e);
exitStatusCode = -1;
@@ -82,7 +82,7 @@ public abstract class AbstractYarnTask extends AbstractTask {
cancel = true;
// cancel process
shellCommandExecutor.cancelApplication();
- TaskInstance taskInstance = processDao.findTaskInstanceById(taskProps.getTaskInstId());
+ TaskInstance taskInstance = processService.findTaskInstanceById(taskProps.getTaskInstId());
if (status && taskInstance != null){
ProcessUtils.killYarnJob(taskInstance);
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java
index 67deb7a3fa..ad62b77655 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java
@@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.server.worker.task;
import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.utils.EnumUtils;
+import org.apache.dolphinscheduler.server.worker.task.conditions.ConditionsTask;
import org.apache.dolphinscheduler.server.worker.task.dependent.DependentTask;
import org.apache.dolphinscheduler.server.worker.task.datax.DataxTask;
import org.apache.dolphinscheduler.server.worker.task.flink.FlinkTask;
@@ -29,6 +30,7 @@ import org.apache.dolphinscheduler.server.worker.task.python.PythonTask;
import org.apache.dolphinscheduler.server.worker.task.shell.ShellTask;
import org.apache.dolphinscheduler.server.worker.task.spark.SparkTask;
import org.apache.dolphinscheduler.server.worker.task.sql.SqlTask;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopTask;
import org.slf4j.Logger;
/**
@@ -68,6 +70,10 @@ public class TaskManager {
return new HttpTask(props, logger);
case DATAX:
return new DataxTask(props, logger);
+ case SQOOP:
+ return new SqoopTask(props, logger);
+ case CONDITIONS:
+ return new ConditionsTask(props, logger);
default:
logger.error("unsupport task type: {}", taskType);
throw new IllegalArgumentException("not support task type");
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/conditions/ConditionsTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/conditions/ConditionsTask.java
new file mode 100644
index 0000000000..cbe82ce20a
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/conditions/ConditionsTask.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.conditions;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.DependResult;
+import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.common.model.DependentItem;
+import org.apache.dolphinscheduler.common.model.DependentTaskModel;
+import org.apache.dolphinscheduler.common.task.AbstractParameters;
+import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
+import org.apache.dolphinscheduler.common.utils.DependentUtils;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
+import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class ConditionsTask extends AbstractTask {
+
+
+ /**
+ * dependent parameters
+ */
+ private DependentParameters dependentParameters;
+
+ /**
+ * process dao
+ */
+ private ProcessService processService;
+
+ /**
+ * taskInstance
+ */
+ private TaskInstance taskInstance;
+
+ /**
+ * processInstance
+ */
+ private ProcessInstance processInstance;
+
+ /**
+ *
+ */
+ private Map completeTaskList = new ConcurrentHashMap<>();
+
+ /**
+ * constructor
+ *
+ * @param taskProps task props
+ * @param logger logger
+ */
+ public ConditionsTask(TaskProps taskProps, Logger logger) {
+ super(taskProps, logger);
+ }
+
+ @Override
+ public void init() throws Exception {
+ logger.info("conditions task initialize");
+
+ this.processService = SpringApplicationContext.getBean(ProcessService.class);
+
+ this.dependentParameters = JSONUtils.parseObject(this.taskProps.getDependence(), DependentParameters.class);
+
+ this.taskInstance = processService.findTaskInstanceById(taskProps.getTaskInstId());
+
+ if(taskInstance == null){
+ throw new Exception("cannot find the task instance!");
+ }
+
+ List taskInstanceList = processService.findValidTaskListByProcessId(taskInstance.getProcessInstanceId());
+ for(TaskInstance task : taskInstanceList){
+ this.completeTaskList.putIfAbsent(task.getName(), task.getState());
+ }
+ }
+
+ @Override
+ public void handle() throws Exception {
+
+ String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, taskProps.getTaskAppId());
+ Thread.currentThread().setName(threadLoggerInfoName);
+
+ List modelResultList = new ArrayList<>();
+ for(DependentTaskModel dependentTaskModel : dependentParameters.getDependTaskList()){
+
+ List itemDependResult = new ArrayList<>();
+ for(DependentItem item : dependentTaskModel.getDependItemList()){
+ itemDependResult.add(getDependResultForItem(item));
+ }
+ DependResult modelResult = DependentUtils.getDependResultForRelation(dependentTaskModel.getRelation(), itemDependResult);
+ modelResultList.add(modelResult);
+ }
+ DependResult result = DependentUtils.getDependResultForRelation(
+ dependentParameters.getRelation(), modelResultList
+ );
+ logger.info("the conditions task depend result : {}", result);
+ exitStatusCode = (result == DependResult.SUCCESS) ?
+ Constants.EXIT_CODE_SUCCESS : Constants.EXIT_CODE_FAILURE;
+ }
+
+ private DependResult getDependResultForItem(DependentItem item){
+
+ DependResult dependResult = DependResult.SUCCESS;
+ if(!completeTaskList.containsKey(item.getDepTasks())){
+ logger.info("depend item: {} have not completed yet.", item.getDepTasks());
+ dependResult = DependResult.FAILED;
+ return dependResult;
+ }
+ ExecutionStatus executionStatus = completeTaskList.get(item.getDepTasks());
+ if(executionStatus != item.getStatus()){
+ logger.info("depend item : {} expect status: {}, actual status: {}" ,item.getDepTasks(), item.getStatus().toString(), executionStatus.toString());
+ dependResult = DependResult.FAILED;
+ }
+ logger.info("depend item: {}, depend result: {}",
+ item.getDepTasks(), dependResult);
+ return dependResult;
+ }
+
+ @Override
+ public AbstractParameters getParameters() {
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
index 0de2bbc7c6..ef941cd062 100755
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
@@ -39,23 +39,23 @@ import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
-import org.apache.dolphinscheduler.common.job.db.BaseDataSource;
-import org.apache.dolphinscheduler.common.job.db.DataSourceFactory;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.datax.DataxParameters;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
-import org.apache.dolphinscheduler.dao.ProcessDao;
+import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
+import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.server.utils.DataxUtils;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import com.alibaba.druid.sql.ast.SQLStatement;
@@ -106,9 +106,9 @@ public class DataxTask extends AbstractTask {
private ShellCommandExecutor shellCommandExecutor;
/**
- * process database access
+ * process dao
*/
- private ProcessDao processDao;
+ private ProcessService processService;
/**
* constructor
@@ -128,7 +128,7 @@ public class DataxTask extends AbstractTask {
props.getTaskInstId(), props.getTenantCode(), props.getEnvFile(), props.getTaskStartTime(),
props.getTaskTimeout(), logger);
- this.processDao = SpringApplicationContext.getBean(ProcessDao.class);
+ this.processService = SpringApplicationContext.getBean(ProcessService.class);
}
/**
@@ -160,7 +160,7 @@ public class DataxTask extends AbstractTask {
// run datax process
String jsonFilePath = buildDataxJsonFile();
String shellCommandFilePath = buildShellCommandFile(jsonFilePath);
- exitStatusCode = shellCommandExecutor.run(shellCommandFilePath, processDao);
+ exitStatusCode = shellCommandExecutor.run(shellCommandFilePath, processService);
}
catch (Exception e) {
exitStatusCode = -1;
@@ -220,11 +220,11 @@ public class DataxTask extends AbstractTask {
*/
private List buildDataxJobContentJson()
throws SQLException {
- DataSource dataSource = processDao.findDataSourceById(dataXParameters.getDataSource());
+ DataSource dataSource = processService.findDataSourceById(dataXParameters.getDataSource());
BaseDataSource dataSourceCfg = DataSourceFactory.getDatasource(dataSource.getType(),
dataSource.getConnectionParams());
- DataSource dataTarget = processDao.findDataSourceById(dataXParameters.getDataTarget());
+ DataSource dataTarget = processService.findDataSourceById(dataXParameters.getDataTarget());
BaseDataSource dataTargetCfg = DataSourceFactory.getDatasource(dataTarget.getType(),
dataTarget.getConnectionParams());
@@ -355,7 +355,7 @@ public class DataxTask extends AbstractTask {
String dataxCommand = sbr.toString();
// find process instance by task id
- ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId());
+ ProcessInstance processInstance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId());
// combining local and global parameters
Map paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(),
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java
index 4be65ed49d..b08cabc2e9 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java
@@ -23,10 +23,10 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.model.DateInterval;
import org.apache.dolphinscheduler.common.model.DependentItem;
import org.apache.dolphinscheduler.common.utils.DependentUtils;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,9 +37,9 @@ import java.util.*;
*/
public class DependentExecute {
/**
- * process dao
+ * process service
*/
- private final ProcessDao processDao = SpringApplicationContext.getBean(ProcessDao.class);
+ private final ProcessService processService = SpringApplicationContext.getBean(ProcessService.class);
/**
* depend item list
@@ -108,7 +108,7 @@ public class DependentExecute {
result = getDependResultByState(processInstance.getState());
}else{
TaskInstance taskInstance = null;
- List taskInstanceList = processDao.findValidTaskListByProcessId(processInstance.getId());
+ List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId());
for(TaskInstance task : taskInstanceList){
if(task.getName().equals(dependentItem.getDepTasks())){
@@ -141,16 +141,16 @@ public class DependentExecute {
*/
private ProcessInstance findLastProcessInterval(int definitionId, DateInterval dateInterval) {
- ProcessInstance runningProcess = processDao.findLastRunningProcess(definitionId, dateInterval);
+ ProcessInstance runningProcess = processService.findLastRunningProcess(definitionId, dateInterval);
if(runningProcess != null){
return runningProcess;
}
- ProcessInstance lastSchedulerProcess = processDao.findLastSchedulerProcessInterval(
+ ProcessInstance lastSchedulerProcess = processService.findLastSchedulerProcessInterval(
definitionId, dateInterval
);
- ProcessInstance lastManualProcess = processDao.findLastManualProcessInterval(
+ ProcessInstance lastManualProcess = processService.findLastManualProcessInterval(
definitionId, dateInterval
);
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java
index 9af29e01dd..f074d57e6c 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java
@@ -25,11 +25,11 @@ import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.DependentUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import java.util.*;
@@ -63,9 +63,9 @@ public class DependentTask extends AbstractTask {
private Date dependentDate;
/**
- * process dao
+ * process service
*/
- private ProcessDao processDao;
+ private ProcessService processService;
/**
* constructor
@@ -88,7 +88,7 @@ public class DependentTask extends AbstractTask {
taskModel.getDependItemList(), taskModel.getRelation()));
}
- this.processDao = SpringApplicationContext.getBean(ProcessDao.class);
+ this.processService = SpringApplicationContext.getBean(ProcessService.class);
if(taskProps.getScheduleTime() != null){
this.dependentDate = taskProps.getScheduleTime();
@@ -107,7 +107,7 @@ public class DependentTask extends AbstractTask {
try{
TaskInstance taskInstance = null;
while(Stopper.isRunning()){
- taskInstance = processDao.findTaskInstanceById(this.taskProps.getTaskInstId());
+ taskInstance = processService.findTaskInstanceById(this.taskProps.getTaskInstId());
if(taskInstance == null){
exitStatusCode = -1;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java
index 0fa9e11ce5..c562fbe4dd 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/flink/FlinkTask.java
@@ -68,7 +68,7 @@ public class FlinkTask extends AbstractYarnTask {
if (StringUtils.isNotEmpty(flinkParameters.getMainArgs())) {
String args = flinkParameters.getMainArgs();
// get process instance by task instance id
- ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId());
+ ProcessInstance processInstance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId());
/**
* combining local and global parameters
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java
index 97e6cb7bee..c925f90b9e 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java
@@ -28,13 +28,13 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.http.HttpParameters;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
import org.apache.dolphinscheduler.common.utils.StringUtils;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.http.HttpEntity;
import org.apache.http.ParseException;
import org.apache.http.client.config.RequestConfig;
@@ -66,9 +66,9 @@ public class HttpTask extends AbstractTask {
private HttpParameters httpParameters;
/**
- * process database access
+ * process service
*/
- private ProcessDao processDao;
+ private ProcessService processService;
/**
* Convert mill seconds to second unit
@@ -92,7 +92,7 @@ public class HttpTask extends AbstractTask {
*/
public HttpTask(TaskProps props, Logger logger) {
super(props, logger);
- this.processDao = SpringApplicationContext.getBean(ProcessDao.class);
+ this.processService = SpringApplicationContext.getBean(ProcessService.class);
}
@Override
@@ -138,7 +138,7 @@ public class HttpTask extends AbstractTask {
*/
protected CloseableHttpResponse sendRequest(CloseableHttpClient client) throws IOException {
RequestBuilder builder = createRequestBuilder();
- ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId());
+ ProcessInstance processInstance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId());
Map paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(),
taskProps.getDefinedParams(),
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java
index 9b4952bbd2..fb881453e9 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java
@@ -22,19 +22,19 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DataType;
import org.apache.dolphinscheduler.common.enums.Direct;
import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy;
-import org.apache.dolphinscheduler.common.job.db.BaseDataSource;
-import org.apache.dolphinscheduler.common.job.db.DataSourceFactory;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
-import org.apache.dolphinscheduler.dao.ProcessDao;
+import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
+import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import java.sql.*;
@@ -56,9 +56,9 @@ public class ProcedureTask extends AbstractTask {
private ProcedureParameters procedureParameters;
/**
- * process database access
+ * process service
*/
- private ProcessDao processDao;
+ private ProcessService processService;
/**
* base datasource
@@ -82,7 +82,7 @@ public class ProcedureTask extends AbstractTask {
throw new RuntimeException("procedure task params is not valid");
}
- this.processDao = SpringApplicationContext.getBean(ProcessDao.class);
+ this.processService = SpringApplicationContext.getBean(ProcessService.class);
}
@Override
@@ -97,7 +97,7 @@ public class ProcedureTask extends AbstractTask {
procedureParameters.getMethod(),
procedureParameters.getLocalParams());
- DataSource dataSource = processDao.findDataSourceById(procedureParameters.getDatasource());
+ DataSource dataSource = processService.findDataSourceById(procedureParameters.getDatasource());
if (dataSource == null){
logger.error("datasource not exists");
exitStatusCode = -1;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java
index 585d62f154..fc212f866b 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java
@@ -22,12 +22,12 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.python.PythonParameters;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.PythonCommandExecutor;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import java.util.Map;
@@ -53,9 +53,9 @@ public class PythonTask extends AbstractTask {
private PythonCommandExecutor pythonCommandExecutor;
/**
- * process database access
+ * process service
*/
- private ProcessDao processDao;
+ private ProcessService processService;
/**
* constructor
@@ -76,7 +76,7 @@ public class PythonTask extends AbstractTask {
taskProps.getTaskStartTime(),
taskProps.getTaskTimeout(),
logger);
- this.processDao = SpringApplicationContext.getBean(ProcessDao.class);
+ this.processService = SpringApplicationContext.getBean(ProcessService.class);
}
@Override
@@ -94,7 +94,7 @@ public class PythonTask extends AbstractTask {
public void handle() throws Exception {
try {
// construct process
- exitStatusCode = pythonCommandExecutor.run(buildCommand(), processDao);
+ exitStatusCode = pythonCommandExecutor.run(buildCommand(), processService);
} catch (Exception e) {
logger.error("python task failure", e);
exitStatusCode = -1;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
index 789a0c5302..5704c8052e 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
@@ -23,12 +23,12 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.shell.ShellParameters;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import java.io.File;
@@ -64,7 +64,7 @@ public class ShellTask extends AbstractTask {
/**
* process database access
*/
- private ProcessDao processDao;
+ private ProcessService processService;
/**
* constructor
@@ -84,7 +84,7 @@ public class ShellTask extends AbstractTask {
taskProps.getTaskStartTime(),
taskProps.getTaskTimeout(),
logger);
- this.processDao = SpringApplicationContext.getBean(ProcessDao.class);
+ this.processService = SpringApplicationContext.getBean(ProcessService.class);
}
@Override
@@ -102,7 +102,7 @@ public class ShellTask extends AbstractTask {
public void handle() throws Exception {
try {
// construct process
- exitStatusCode = shellCommandExecutor.run(buildCommand(), processDao);
+ exitStatusCode = shellCommandExecutor.run(buildCommand(), processService);
} catch (Exception e) {
logger.error("shell task failure", e);
exitStatusCode = -1;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
index eba05a0d21..aae11f5530 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
@@ -27,8 +27,6 @@ import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy;
import org.apache.dolphinscheduler.common.enums.UdfType;
-import org.apache.dolphinscheduler.common.job.db.BaseDataSource;
-import org.apache.dolphinscheduler.common.job.db.DataSourceFactory;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.sql.SqlBinds;
@@ -36,16 +34,19 @@ import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
import org.apache.dolphinscheduler.common.task.sql.SqlType;
import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.dao.AlertDao;
-import org.apache.dolphinscheduler.dao.ProcessDao;
+import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
+import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.dao.permission.PermissionCheck;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.utils.UDFUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.permission.PermissionCheck;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import java.sql.*;
@@ -67,9 +68,9 @@ public class SqlTask extends AbstractTask {
private SqlParameters sqlParameters;
/**
- * process database access
+ * process service
*/
- private ProcessDao processDao;
+ private ProcessService processService;
/**
* alert dao
@@ -96,7 +97,7 @@ public class SqlTask extends AbstractTask {
if (!sqlParameters.checkParameters()) {
throw new RuntimeException("sql task params is not valid");
}
- this.processDao = SpringApplicationContext.getBean(ProcessDao.class);
+ this.processService = SpringApplicationContext.getBean(ProcessService.class);
this.alertDao = SpringApplicationContext.getBean(AlertDao.class);
}
@@ -122,7 +123,7 @@ public class SqlTask extends AbstractTask {
return;
}
- dataSource= processDao.findDataSourceById(sqlParameters.getDatasource());
+ dataSource= processService.findDataSourceById(sqlParameters.getDatasource());
// data source is null
if (dataSource == null){
@@ -171,7 +172,7 @@ public class SqlTask extends AbstractTask {
}
// check udf permission
checkUdfPermission(ArrayUtils.toObject(idsArray));
- List udfFuncList = processDao.queryUdfFunListByids(idsArray);
+ List udfFuncList = processService.queryUdfFunListByids(idsArray);
createFuncs = UDFUtils.createFuncs(udfFuncList, taskProps.getTenantCode(), logger);
}
@@ -383,7 +384,7 @@ public class SqlTask extends AbstractTask {
public void sendAttachment(String title,String content){
// process instance
- ProcessInstance instance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId());
+ ProcessInstance instance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId());
List users = alertDao.queryUserByAlertGroupId(instance.getWarningGroupId());
@@ -470,10 +471,10 @@ public class SqlTask extends AbstractTask {
*/
private void checkUdfPermission(Integer[] udfFunIds) throws Exception{
// process instance
- ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId());
+ ProcessInstance processInstance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId());
int userId = processInstance.getExecutorId();
- PermissionCheck permissionCheckUdf = new PermissionCheck(AuthorizationType.UDF,processDao,udfFunIds,userId,logger);
+ PermissionCheck permissionCheckUdf = new PermissionCheck(AuthorizationType.UDF, processService,udfFunIds,userId,logger);
permissionCheckUdf.checkPermission();
}
@@ -484,10 +485,10 @@ public class SqlTask extends AbstractTask {
*/
private void checkDataSourcePermission(int dataSourceId) throws Exception{
// process instance
- ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskProps.getTaskInstId());
+ ProcessInstance processInstance = processService.findProcessInstanceByTaskId(taskProps.getTaskInstId());
int userId = processInstance.getExecutorId();
- PermissionCheck permissionCheckDataSource = new PermissionCheck(AuthorizationType.DATASOURCE,processDao,new Integer[]{dataSourceId},userId,logger);
+ PermissionCheck permissionCheckDataSource = new PermissionCheck(AuthorizationType.DATASOURCE, processService,new Integer[]{dataSourceId},userId,logger);
permissionCheckDataSource.checkPermission();
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java
new file mode 100644
index 0000000000..64bc7924d2
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTask.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.sqoop;
+
+import com.alibaba.fastjson.JSON;
+import org.apache.dolphinscheduler.common.process.Property;
+import org.apache.dolphinscheduler.common.task.AbstractParameters;
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.common.utils.ParameterUtils;
+import org.apache.dolphinscheduler.server.utils.ParamUtils;
+import org.apache.dolphinscheduler.server.worker.task.AbstractYarnTask;
+import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator;
+import org.slf4j.Logger;
+import java.util.Map;
+
+/**
+ * sqoop task extends the shell task
+ */
+public class SqoopTask extends AbstractYarnTask {
+
+ private SqoopParameters sqoopParameters;
+
+ public SqoopTask(TaskProps props, Logger logger){
+ super(props,logger);
+ }
+
+ @Override
+ public void init() throws Exception {
+ logger.info("sqoop task params {}", taskProps.getTaskParams());
+ sqoopParameters =
+ JSON.parseObject(taskProps.getTaskParams(),SqoopParameters.class);
+ if (!sqoopParameters.checkParameters()) {
+ throw new RuntimeException("sqoop task params is not valid");
+ }
+
+ }
+
+ @Override
+ protected String buildCommand() throws Exception {
+ //get sqoop scripts
+ SqoopJobGenerator generator = new SqoopJobGenerator();
+ String script = generator.generateSqoopJob(sqoopParameters);
+
+ Map paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(),
+ taskProps.getDefinedParams(),
+ sqoopParameters.getLocalParametersMap(),
+ taskProps.getCmdTypeIfComplement(),
+ taskProps.getScheduleTime());
+
+ if(paramsMap != null){
+ String resultScripts = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
+ logger.info("sqoop script: {}", resultScripts);
+ return resultScripts;
+ }
+
+ return null;
+ }
+
+ @Override
+ public AbstractParameters getParameters() {
+ return sqoopParameters;
+ }
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/CommonGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/CommonGenerator.java
new file mode 100644
index 0000000000..4944bac5ba
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/CommonGenerator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.sqoop.generator;
+
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * common script generator
+ */
+public class CommonGenerator {
+
+ private Logger logger = LoggerFactory.getLogger(getClass());
+
+ public String generate(SqoopParameters sqoopParameters) {
+ StringBuilder result = new StringBuilder();
+ try{
+ result.append("sqoop ")
+ .append(sqoopParameters.getModelType());
+ if(sqoopParameters.getConcurrency() >0){
+ result.append(" -m ")
+ .append(sqoopParameters.getConcurrency());
+ }
+ }catch (Exception e){
+ logger.error(e.getMessage());
+ }
+
+ return result.toString();
+ }
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ISourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ISourceGenerator.java
new file mode 100644
index 0000000000..6c1d1fdca8
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ISourceGenerator.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.sqoop.generator;
+
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+
+/**
+ * Source Generator Interface
+ */
+public interface ISourceGenerator {
+
+ /**
+ * generate the source script
+ * @param sqoopParameters sqoop params
+ * @return
+ */
+ String generate(SqoopParameters sqoopParameters);
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ITargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ITargetGenerator.java
new file mode 100644
index 0000000000..be307af5f2
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/ITargetGenerator.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.sqoop.generator;
+
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+
+/**
+ * Target Generator Interface
+ */
+public interface ITargetGenerator {
+
+ /**
+ * generate the target script
+ * @param sqoopParameters sqoop params
+ * @return
+ */
+ String generate(SqoopParameters sqoopParameters);
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/SqoopJobGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/SqoopJobGenerator.java
new file mode 100644
index 0000000000..24c76e027d
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/SqoopJobGenerator.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.sqoop.generator;
+
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources.HdfsSourceGenerator;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources.HiveSourceGenerator;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources.MysqlSourceGenerator;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets.HdfsTargetGenerator;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets.HiveTargetGenerator;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets.MysqlTargetGenerator;
+
+/**
+ * Sqoop Job Scripts Generator
+ */
+public class SqoopJobGenerator {
+
+ private static final String MYSQL = "MYSQL";
+ private static final String HIVE = "HIVE";
+ private static final String HDFS = "HDFS";
+
+ /**
+ * target script generator
+ */
+ private ITargetGenerator targetGenerator;
+ /**
+ * source script generator
+ */
+ private ISourceGenerator sourceGenerator;
+ /**
+ * common script generator
+ */
+ private CommonGenerator commonGenerator;
+
+ public SqoopJobGenerator(){
+ commonGenerator = new CommonGenerator();
+ }
+
+ private void createSqoopJobGenerator(String sourceType,String targetType){
+ sourceGenerator = createSourceGenerator(sourceType);
+ targetGenerator = createTargetGenerator(targetType);
+ }
+
+ /**
+ * get the final sqoop scripts
+ * @param sqoopParameters
+ * @return
+ */
+ public String generateSqoopJob(SqoopParameters sqoopParameters){
+ createSqoopJobGenerator(sqoopParameters.getSourceType(),sqoopParameters.getTargetType());
+ if(sourceGenerator == null || targetGenerator == null){
+ return null;
+ }
+
+ return commonGenerator.generate(sqoopParameters)
+ + sourceGenerator.generate(sqoopParameters)
+ + targetGenerator.generate(sqoopParameters);
+ }
+
+ /**
+ * get the source generator
+ * @param sourceType
+ * @return
+ */
+ private ISourceGenerator createSourceGenerator(String sourceType){
+ switch (sourceType){
+ case MYSQL:
+ return new MysqlSourceGenerator();
+ case HIVE:
+ return new HiveSourceGenerator();
+ case HDFS:
+ return new HdfsSourceGenerator();
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * get the target generator
+ * @param targetType
+ * @return
+ */
+ private ITargetGenerator createTargetGenerator(String targetType){
+ switch (targetType){
+ case MYSQL:
+ return new MysqlTargetGenerator();
+ case HIVE:
+ return new HiveTargetGenerator();
+ case HDFS:
+ return new HdfsTargetGenerator();
+ default:
+ return null;
+ }
+ }
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java
new file mode 100644
index 0000000000..47b01363e6
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HdfsSourceGenerator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources;
+
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHdfsParameter;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * hdfs source generator
+ */
+public class HdfsSourceGenerator implements ISourceGenerator {
+
+ private Logger logger = LoggerFactory.getLogger(getClass());
+
+ @Override
+ public String generate(SqoopParameters sqoopParameters) {
+ StringBuilder result = new StringBuilder();
+ try{
+ SourceHdfsParameter sourceHdfsParameter
+ = JSONUtils.parseObject(sqoopParameters.getSourceParams(),SourceHdfsParameter.class);
+
+ if(sourceHdfsParameter != null){
+ if(StringUtils.isNotEmpty(sourceHdfsParameter.getExportDir())){
+ result.append(" --export-dir ")
+ .append(sourceHdfsParameter.getExportDir());
+ }else{
+ throw new Exception("--export-dir is null");
+ }
+
+ }
+ }catch (Exception e){
+ logger.error("get hdfs source failed",e);
+ }
+
+ return result.toString();
+ }
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java
new file mode 100644
index 0000000000..91363e296a
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/HiveSourceGenerator.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHiveParameter;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * hive source generator
+ */
+public class HiveSourceGenerator implements ISourceGenerator {
+
+ private Logger logger = LoggerFactory.getLogger(getClass());
+
+ @Override
+ public String generate(SqoopParameters sqoopParameters) {
+ StringBuilder sb = new StringBuilder();
+ try{
+ SourceHiveParameter sourceHiveParameter
+ = JSONUtils.parseObject(sqoopParameters.getSourceParams(),SourceHiveParameter.class);
+ if(sourceHiveParameter != null){
+ if(StringUtils.isNotEmpty(sourceHiveParameter.getHiveDatabase())){
+ sb.append(" --hcatalog-database ").append(sourceHiveParameter.getHiveDatabase());
+ }
+
+ if(StringUtils.isNotEmpty(sourceHiveParameter.getHiveTable())){
+ sb.append(" --hcatalog-table ").append(sourceHiveParameter.getHiveTable());
+ }
+
+ if(StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionKey())&&
+ StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionValue())){
+ sb.append(" --hcatalog-partition-keys ").append(sourceHiveParameter.getHivePartitionKey())
+ .append(" --hcatalog-partition-values ").append(sourceHiveParameter.getHivePartitionValue());
+ }
+ }
+ }catch (Exception e){
+ logger.error(e.getMessage());
+ }
+
+ return sb.toString();
+ }
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java
new file mode 100644
index 0000000000..050fef7cc7
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.sources;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.enums.QueryType;
+import org.apache.dolphinscheduler.common.process.Property;
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
+import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.dao.entity.DataSource;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ISourceGenerator;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * mysql source generator
+ */
+public class MysqlSourceGenerator implements ISourceGenerator {
+
+ private Logger logger = LoggerFactory.getLogger(getClass());
+
+ @Override
+ public String generate(SqoopParameters sqoopParameters) {
+ StringBuilder result = new StringBuilder();
+ try {
+ SourceMysqlParameter sourceMysqlParameter
+ = JSONUtils.parseObject(sqoopParameters.getSourceParams(),SourceMysqlParameter.class);
+
+ if(sourceMysqlParameter != null){
+ ProcessService processService = SpringApplicationContext.getBean(ProcessService.class);
+ DataSource dataSource= processService.findDataSourceById(sourceMysqlParameter.getSrcDatasource());
+ BaseDataSource baseDataSource = DataSourceFactory.getDatasource(dataSource.getType(),
+ dataSource.getConnectionParams());
+ if(baseDataSource != null){
+ result.append(" --connect ")
+ .append(baseDataSource.getJdbcUrl())
+ .append(" --username ")
+ .append(baseDataSource.getUser())
+ .append(" --password ")
+ .append(baseDataSource.getPassword());
+
+ if(sourceMysqlParameter.getSrcQueryType() == QueryType.FORM.ordinal()){
+ if(StringUtils.isNotEmpty(sourceMysqlParameter.getSrcTable())){
+ result.append(" --table ").append(sourceMysqlParameter.getSrcTable());
+ }
+
+ if(StringUtils.isNotEmpty(sourceMysqlParameter.getSrcColumns())){
+ result.append(" --columns ").append(sourceMysqlParameter.getSrcColumns());
+ }
+
+ }else if(sourceMysqlParameter.getSrcQueryType() == QueryType.SQL.ordinal()){
+ if(StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())){
+
+ String srcQuery = sourceMysqlParameter.getSrcQuerySql();
+ if(srcQuery.toLowerCase().contains("where")){
+ srcQuery += " AND "+"$CONDITIONS";
+ }else{
+ srcQuery += " WHERE $CONDITIONS";
+ }
+ result.append(" --query \'"+srcQuery+"\'");
+ }
+ }
+
+ List mapColumnHive = sourceMysqlParameter.getMapColumnHive();
+
+ if(mapColumnHive != null && !mapColumnHive.isEmpty()){
+ String columnMap = "";
+ for(Property item:mapColumnHive){
+ columnMap = item.getProp()+"="+ item.getValue()+",";
+ }
+
+ if(StringUtils.isNotEmpty(columnMap)){
+ result.append(" --map-column-hive ")
+ .append(columnMap.substring(0,columnMap.length()-1));
+ }
+ }
+
+ List mapColumnJava = sourceMysqlParameter.getMapColumnJava();
+
+ if(mapColumnJava != null && !mapColumnJava.isEmpty()){
+ String columnMap = "";
+ for(Property item:mapColumnJava){
+ columnMap = item.getProp()+"="+ item.getValue()+",";
+ }
+
+ if(StringUtils.isNotEmpty(columnMap)){
+ result.append(" --map-column-java ")
+ .append(columnMap.substring(0,columnMap.length()-1));
+ }
+ }
+ }
+ }
+ }catch (Exception e){
+ logger.error(e.getMessage());
+ }
+
+ return result.toString();
+ }
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java
new file mode 100644
index 0000000000..411e9b4450
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HdfsTargetGenerator.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHdfsParameter;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ITargetGenerator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * hdfs target generator
+ */
+public class HdfsTargetGenerator implements ITargetGenerator {
+
+ private Logger logger = LoggerFactory.getLogger(getClass());
+
+ @Override
+ public String generate(SqoopParameters sqoopParameters) {
+ StringBuilder result = new StringBuilder();
+ try{
+ TargetHdfsParameter targetHdfsParameter =
+ JSONUtils.parseObject(sqoopParameters.getTargetParams(),TargetHdfsParameter.class);
+
+ if(targetHdfsParameter != null){
+
+ if(StringUtils.isNotEmpty(targetHdfsParameter.getTargetPath())){
+ result.append(" --target-dir ").append(targetHdfsParameter.getTargetPath());
+ }
+
+ if(StringUtils.isNotEmpty(targetHdfsParameter.getCompressionCodec())){
+ result.append(" --compression-codec ").append(targetHdfsParameter.getCompressionCodec());
+ }
+
+ if(StringUtils.isNotEmpty(targetHdfsParameter.getFileType())){
+ result.append(" ").append(targetHdfsParameter.getFileType());
+ }
+
+ if(targetHdfsParameter.isDeleteTargetDir()){
+ result.append(" --delete-target-dir");
+ }
+
+ if(StringUtils.isNotEmpty(targetHdfsParameter.getFieldsTerminated())){
+ result.append(" --fields-terminated-by '").append(targetHdfsParameter.getFieldsTerminated()).append("'");
+ }
+
+ if(StringUtils.isNotEmpty(targetHdfsParameter.getLinesTerminated())){
+ result.append(" --lines-terminated-by '").append(targetHdfsParameter.getLinesTerminated()).append("'");
+ }
+
+ result.append(" --null-non-string 'NULL' --null-string 'NULL'");
+ }
+ }catch(Exception e){
+ logger.error(e.getMessage());
+ }
+
+ return result.toString();
+ }
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java
new file mode 100644
index 0000000000..ad59173ad0
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHiveParameter;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ITargetGenerator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * hive target generator
+ */
+public class HiveTargetGenerator implements ITargetGenerator {
+
+ private Logger logger = LoggerFactory.getLogger(getClass());
+
+ @Override
+ public String generate(SqoopParameters sqoopParameters) {
+
+ StringBuilder result = new StringBuilder();
+
+ try{
+ TargetHiveParameter targetHiveParameter =
+ JSONUtils.parseObject(sqoopParameters.getTargetParams(),TargetHiveParameter.class);
+ if(targetHiveParameter != null){
+
+ result.append(" --hive-import ");
+
+ if(StringUtils.isNotEmpty(targetHiveParameter.getHiveDatabase())&&
+ StringUtils.isNotEmpty(targetHiveParameter.getHiveTable())){
+ result.append(" --hive-table ")
+ .append(targetHiveParameter.getHiveDatabase())
+ .append(".")
+ .append(targetHiveParameter.getHiveTable());
+ }
+
+ if(targetHiveParameter.isCreateHiveTable()){
+ result.append(" --create-hive-table");
+ }
+
+ if(targetHiveParameter.isDropDelimiter()){
+ result.append(" --hive-drop-import-delims");
+ }
+
+ if(targetHiveParameter.isHiveOverWrite()){
+ result.append(" --hive-overwrite -delete-target-dir");
+ }
+
+ if(StringUtils.isNotEmpty(targetHiveParameter.getReplaceDelimiter())){
+ result.append(" --hive-delims-replacement ").append(targetHiveParameter.getReplaceDelimiter());
+ }
+
+ if(StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionKey())&&
+ StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionValue())){
+ result.append(" --hive-partition-key ")
+ .append(targetHiveParameter.getHivePartitionKey())
+ .append(" --hive-partition-value ")
+ .append(targetHiveParameter.getHivePartitionValue());
+ }
+
+ }
+ }catch(Exception e){
+ logger.error(e.getMessage());
+ }
+
+ return result.toString();
+ }
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java
new file mode 100644
index 0000000000..0733338812
--- /dev/null
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.sqoop.generator.targets;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParameter;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
+import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
+import org.apache.dolphinscheduler.dao.entity.DataSource;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.ITargetGenerator;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * mysql target generator
+ */
+public class MysqlTargetGenerator implements ITargetGenerator {
+
+ private Logger logger = LoggerFactory.getLogger(getClass());
+
+ @Override
+ public String generate(SqoopParameters sqoopParameters) {
+
+ StringBuilder result = new StringBuilder();
+ try{
+
+ TargetMysqlParameter targetMysqlParameter =
+ JSONUtils.parseObject(sqoopParameters.getTargetParams(),TargetMysqlParameter.class);
+
+ if(targetMysqlParameter != null && targetMysqlParameter.getTargetDatasource() != 0){
+
+ ProcessService processService = SpringApplicationContext.getBean(ProcessService.class);
+ DataSource dataSource= processService.findDataSourceById(targetMysqlParameter.getTargetDatasource());
+ // get datasource
+ BaseDataSource baseDataSource = DataSourceFactory.getDatasource(dataSource.getType(),
+ dataSource.getConnectionParams());
+
+ if(baseDataSource != null){
+ result.append(" --connect ")
+ .append(baseDataSource.getJdbcUrl())
+ .append(" --username ")
+ .append(baseDataSource.getUser())
+ .append(" --password ")
+ .append(baseDataSource.getPassword())
+ .append(" --table ")
+ .append(targetMysqlParameter.getTargetTable());
+
+ if(StringUtils.isNotEmpty(targetMysqlParameter.getTargetColumns())){
+ result.append(" --columns ").append(targetMysqlParameter.getTargetColumns());
+ }
+
+ if(StringUtils.isNotEmpty(targetMysqlParameter.getFieldsTerminated())){
+ result.append(" --fields-terminated-by '").append(targetMysqlParameter.getFieldsTerminated()).append("'");
+ }
+
+ if(StringUtils.isNotEmpty(targetMysqlParameter.getLinesTerminated())){
+ result.append(" --lines-terminated-by '").append(targetMysqlParameter.getLinesTerminated()).append("'");
+ }
+
+ if(targetMysqlParameter.isUpdate()){
+ if(StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey())&&
+ StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())){
+ result.append(" --update-key ").append(targetMysqlParameter.getTargetUpdateKey())
+ .append(" --update-mode ").append(targetMysqlParameter.getTargetUpdateMode());
+ }
+ }
+ }
+ }
+ }catch (Exception e){
+ logger.error(e.getMessage());
+ }
+
+ return result.toString();
+ }
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java
index c6a71ed066..fe4ec9130a 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java
@@ -21,10 +21,8 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.ZKNodeType;
import org.apache.dolphinscheduler.common.model.Server;
-import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.DaoFactory;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.ProcessUtils;
@@ -32,6 +30,8 @@ import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.curator.utils.ThreadUtils;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.service.zk.AbstractZKClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -70,10 +70,10 @@ public class ZKMasterClient extends AbstractZKClient {
*/
private AlertDao alertDao = null;
/**
- * flow database access
+ * process service
*/
@Autowired
- private ProcessDao processDao;
+ private ProcessService processService;
/**
* default constructor
@@ -374,7 +374,7 @@ public class ZKMasterClient extends AbstractZKClient {
private void failoverWorker(String workerHost, boolean needCheckWorkerAlive) throws Exception {
logger.info("start worker[{}] failover ...", workerHost);
- List needFailoverTaskInstanceList = processDao.queryNeedFailoverTaskInstances(workerHost);
+ List needFailoverTaskInstanceList = processService.queryNeedFailoverTaskInstances(workerHost);
for(TaskInstance taskInstance : needFailoverTaskInstanceList){
if(needCheckWorkerAlive){
if(!checkTaskInstanceNeedFailover(taskInstance)){
@@ -382,7 +382,7 @@ public class ZKMasterClient extends AbstractZKClient {
}
}
- ProcessInstance instance = processDao.findProcessInstanceDetailById(taskInstance.getProcessInstanceId());
+ ProcessInstance instance = processService.findProcessInstanceDetailById(taskInstance.getProcessInstanceId());
if(instance!=null){
taskInstance.setProcessInstance(instance);
}
@@ -390,7 +390,7 @@ public class ZKMasterClient extends AbstractZKClient {
ProcessUtils.killYarnJob(taskInstance);
taskInstance.setState(ExecutionStatus.NEED_FAULT_TOLERANCE);
- processDao.saveTaskInstance(taskInstance);
+ processService.saveTaskInstance(taskInstance);
}
logger.info("end worker[{}] failover ...", workerHost);
}
@@ -403,11 +403,11 @@ public class ZKMasterClient extends AbstractZKClient {
private void failoverMaster(String masterHost) {
logger.info("start master failover ...");
- List needFailoverProcessInstanceList = processDao.queryNeedFailoverProcessInstances(masterHost);
+ List needFailoverProcessInstanceList = processService.queryNeedFailoverProcessInstances(masterHost);
//updateProcessInstance host is null and insert into command
for(ProcessInstance processInstance : needFailoverProcessInstanceList){
- processDao.processNeedFailoverProcessInstances(processInstance);
+ processService.processNeedFailoverProcessInstances(processInstance);
}
logger.info("master failover end");
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java
index 88abfa3071..7ddee3b2a1 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java
@@ -19,9 +19,9 @@ package org.apache.dolphinscheduler.server.zk;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ZKNodeType;
-import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
+import org.apache.dolphinscheduler.service.zk.AbstractZKClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java
index d7c3de13a5..d2a0fb2407 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java
@@ -20,13 +20,13 @@ import com.alibaba.fastjson.JSONObject;
import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.utils.DateUtils;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.runner.MasterExecThread;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -57,7 +57,7 @@ public class MasterExecThreadTest {
private ProcessInstance processInstance;
- private ProcessDao processDao;
+ private ProcessService processService;
private int processDefinitionId = 1;
@@ -67,7 +67,7 @@ public class MasterExecThreadTest {
@Before
public void init() throws Exception{
- processDao = mock(ProcessDao.class);
+ processService = mock(ProcessService.class);
applicationContext = mock(ApplicationContext.class);
config = new MasterConfig();
@@ -91,7 +91,7 @@ public class MasterExecThreadTest {
processDefinition.setGlobalParamList(Collections.EMPTY_LIST);
Mockito.when(processInstance.getProcessDefinition()).thenReturn(processDefinition);
- masterExecThread = PowerMockito.spy(new MasterExecThread(processInstance, processDao));
+ masterExecThread = PowerMockito.spy(new MasterExecThread(processInstance, processService));
// prepareProcess init dag
Field dag = MasterExecThread.class.getDeclaredField("dag");
dag.setAccessible(true);
@@ -110,12 +110,12 @@ public class MasterExecThreadTest {
@Test
public void testParallelWithOutSchedule() throws ParseException {
try{
- Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList());
+ Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList());
Method method = MasterExecThread.class.getDeclaredMethod("executeComplementProcess");
method.setAccessible(true);
method.invoke(masterExecThread);
// one create save, and 1-30 for next save, and last day 31 no save
- verify(processDao, times(31)).saveProcessInstance(processInstance);
+ verify(processService, times(31)).saveProcessInstance(processInstance);
}catch (Exception e){
e.printStackTrace();
Assert.assertTrue(false);
@@ -129,12 +129,12 @@ public class MasterExecThreadTest {
@Test
public void testParallelWithSchedule() throws ParseException {
try{
- Mockito.when(processDao.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(oneSchedulerList());
+ Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(oneSchedulerList());
Method method = MasterExecThread.class.getDeclaredMethod("executeComplementProcess");
method.setAccessible(true);
method.invoke(masterExecThread);
// one create save, and 15(1 to 31 step 2) for next save, and last day 31 no save
- verify(processDao, times(15)).saveProcessInstance(processInstance);
+ verify(processService, times(15)).saveProcessInstance(processInstance);
}catch (Exception e){
Assert.assertTrue(false);
}
@@ -151,4 +151,5 @@ public class MasterExecThreadTest {
schedulerList.add(schedule);
return schedulerList;
}
+
}
\ No newline at end of file
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java
index 04c844827f..5d4263644b 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java
@@ -20,13 +20,13 @@ import com.alibaba.fastjson.JSONObject;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.model.TaskNode;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskManager;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
@@ -43,11 +43,11 @@ public class ShellCommandExecutorTest {
private static final Logger logger = LoggerFactory.getLogger(ShellCommandExecutorTest.class);
- private ProcessDao processDao = null;
+ private ProcessService processService = null;
@Before
public void before(){
- processDao = SpringApplicationContext.getBean(ProcessDao.class);
+ processService = SpringApplicationContext.getBean(ProcessService.class);
}
@Test
@@ -65,7 +65,7 @@ public class ShellCommandExecutorTest {
- TaskInstance taskInstance = processDao.findTaskInstanceById(7657);
+ TaskInstance taskInstance = processService.findTaskInstanceById(7657);
String taskJson = taskInstance.getTaskJson();
TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class);
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java
index 7da3f710b6..c395eabe51 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java
@@ -21,13 +21,13 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.model.TaskNode;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskManager;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
@@ -44,11 +44,11 @@ public class SqlExecutorTest {
private static final Logger logger = LoggerFactory.getLogger(SqlExecutorTest.class);
- private ProcessDao processDao = null;
+ private ProcessService processService = null;
@Before
public void before(){
- processDao = SpringApplicationContext.getBean(ProcessDao.class);
+ processService = SpringApplicationContext.getBean(ProcessService.class);
}
@Test
@@ -109,7 +109,7 @@ public class SqlExecutorTest {
taskProps.setCmdTypeIfComplement(CommandType.START_PROCESS);
- TaskInstance taskInstance = processDao.findTaskInstanceById(taskInstId);
+ TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId);
String taskJson = taskInstance.getTaskJson();
TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class);
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java
index 7a6073e05d..bd7f27530a 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTaskTest.java
@@ -25,15 +25,15 @@ import java.util.List;
import com.alibaba.fastjson.JSONObject;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DbType;
-import org.apache.dolphinscheduler.common.job.db.BaseDataSource;
-import org.apache.dolphinscheduler.common.job.db.DataSourceFactory;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
-import org.apache.dolphinscheduler.dao.ProcessDao;
+import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
+import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.server.utils.DataxUtils;
import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -53,7 +53,7 @@ public class DataxTaskTest {
private DataxTask dataxTask;
- private ProcessDao processDao;
+ private ProcessService processService;
private ShellCommandExecutor shellCommandExecutor;
@@ -62,13 +62,13 @@ public class DataxTaskTest {
@Before
public void before()
throws Exception {
- processDao = Mockito.mock(ProcessDao.class);
+ processService = Mockito.mock(ProcessService.class);
shellCommandExecutor = Mockito.mock(ShellCommandExecutor.class);
applicationContext = Mockito.mock(ApplicationContext.class);
SpringApplicationContext springApplicationContext = new SpringApplicationContext();
springApplicationContext.setApplicationContext(applicationContext);
- Mockito.when(applicationContext.getBean(ProcessDao.class)).thenReturn(processDao);
+ Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
TaskProps props = new TaskProps();
props.setTaskDir("/tmp");
@@ -83,12 +83,12 @@ public class DataxTaskTest {
dataxTask = PowerMockito.spy(new DataxTask(props, logger));
dataxTask.init();
- Mockito.when(processDao.findDataSourceById(1)).thenReturn(getDataSource());
- Mockito.when(processDao.findDataSourceById(2)).thenReturn(getDataSource());
- Mockito.when(processDao.findProcessInstanceByTaskId(1)).thenReturn(getProcessInstance());
+ Mockito.when(processService.findDataSourceById(1)).thenReturn(getDataSource());
+ Mockito.when(processService.findDataSourceById(2)).thenReturn(getDataSource());
+ Mockito.when(processService.findProcessInstanceByTaskId(1)).thenReturn(getProcessInstance());
String fileName = String.format("%s/%s_node.sh", props.getTaskDir(), props.getTaskAppId());
- Mockito.when(shellCommandExecutor.run(fileName, processDao)).thenReturn(0);
+ Mockito.when(shellCommandExecutor.run(fileName, processService)).thenReturn(0);
}
private DataSource getDataSource() {
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java
new file mode 100644
index 0000000000..f8688e7c0c
--- /dev/null
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.sqoop;
+
+import com.alibaba.fastjson.JSONObject;
+import org.apache.dolphinscheduler.common.enums.DbType;
+import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
+import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHdfsParameter;
+import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHiveParameter;
+import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter;
+import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHdfsParameter;
+import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHiveParameter;
+import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParameter;
+import org.apache.dolphinscheduler.dao.entity.DataSource;
+import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ApplicationContext;
+
+import java.util.*;
+
+/**
+ * sqoop task test
+ */
+@RunWith(MockitoJUnitRunner.Silent.class)
+public class SqoopTaskTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(SqoopTaskTest.class);
+
+ private ProcessService processService;
+ private ApplicationContext applicationContext;
+ private SqoopTask sqoopTask;
+
+ @Before
+ public void before() throws Exception{
+ processService = Mockito.mock(ProcessService.class);
+ Mockito.when(processService.findDataSourceById(2)).thenReturn(getDataSource());
+ applicationContext = Mockito.mock(ApplicationContext.class);
+ SpringApplicationContext springApplicationContext = new SpringApplicationContext();
+ springApplicationContext.setApplicationContext(applicationContext);
+ Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
+
+ TaskProps props = new TaskProps();
+ props.setTaskDir("/tmp");
+ props.setTaskAppId(String.valueOf(System.currentTimeMillis()));
+ props.setTaskInstId(1);
+ props.setTenantCode("1");
+ props.setEnvFile(".dolphinscheduler_env.sh");
+ props.setTaskStartTime(new Date());
+ props.setTaskTimeout(0);
+ props.setTaskParams("{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}");
+
+ sqoopTask = new SqoopTask(props,logger);
+ sqoopTask.init();
+ }
+
+ @Test
+ public void testGenerator(){
+ String data1 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HDFS\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"0\\\",\\\"srcQuerySql\\\":\\\"\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[]}\",\"targetParams\":\"{\\\"targetPath\\\":\\\"/ods/tmp/test/person7\\\",\\\"deleteTargetDir\\\":true,\\\"fileType\\\":\\\"--as-textfile\\\",\\\"compressionCodec\\\":\\\"\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}";
+ SqoopParameters sqoopParameters1 = JSONObject.parseObject(data1,SqoopParameters.class);
+
+ SqoopJobGenerator generator = new SqoopJobGenerator();
+ String script = generator.generateSqoopJob(sqoopParameters1);
+ String expected = "sqoop import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_2 --target-dir /ods/tmp/test/person7 --as-textfile --delete-target-dir --fields-terminated-by '@' --lines-terminated-by '\\n' --null-non-string 'NULL' --null-string 'NULL'";
+ Assert.assertEquals(expected, script);
+
+ String data2 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HDFS\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"exportDir\\\":\\\"/ods/tmp/test/person7\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"id,name,age,sex,create_time\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":true,\\\"targetUpdateKey\\\":\\\"id\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}";
+ SqoopParameters sqoopParameters2 = JSONObject.parseObject(data2,SqoopParameters.class);
+
+ String script2 = generator.generateSqoopJob(sqoopParameters2);
+ String expected2 = "sqoop export -m 1 --export-dir /ods/tmp/test/person7 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --columns id,name,age,sex,create_time --fields-terminated-by '@' --lines-terminated-by '\\n' --update-key id --update-mode allowinsert";
+ Assert.assertEquals(expected2, script2);
+
+ String data3 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HIVE\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-17\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":false,\\\"targetUpdateKey\\\":\\\"\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}";
+ SqoopParameters sqoopParameters3 = JSONObject.parseObject(data3,SqoopParameters.class);
+
+ String script3 = generator.generateSqoopJob(sqoopParameters3);
+ String expected3 = "sqoop export -m 1 --hcatalog-database stg --hcatalog-table person_internal --hcatalog-partition-keys date --hcatalog-partition-values 2020-02-17 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --fields-terminated-by '@' --lines-terminated-by '\\n'";
+ Assert.assertEquals(expected3, script3);
+
+ String data4 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}";
+ SqoopParameters sqoopParameters4 = JSONObject.parseObject(data4,SqoopParameters.class);
+
+ String script4 = generator.generateSqoopJob(sqoopParameters4);
+ String expected4 = "sqoop import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --query 'SELECT * FROM person_2 WHERE $CONDITIONS' --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 --create-hive-table --hive-overwrite -delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16";
+ Assert.assertEquals(expected4, script4);
+
+ }
+
+ private DataSource getDataSource() {
+ DataSource dataSource = new DataSource();
+ dataSource.setType(DbType.MYSQL);
+ dataSource.setConnectionParams(
+ "{\"address\":\"jdbc:mysql://192.168.0.111:3306\",\"database\":\"test\",\"jdbcUrl\":\"jdbc:mysql://192.168.0.111:3306/test\",\"user\":\"kylo\",\"password\":\"123456\"}");
+ dataSource.setUserId(1);
+ return dataSource;
+ }
+
+ @Test
+ public void testGetParameters() {
+ Assert.assertNotNull(sqoopTask.getParameters());
+ }
+
+ /**
+ * Method: init
+ */
+ @Test
+ public void testInit(){
+ try {
+ sqoopTask.init();
+ } catch (Exception e) {
+ Assert.fail(e.getMessage());
+ }
+ }
+
+}
diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml
new file mode 100644
index 0000000000..7d775d5497
--- /dev/null
+++ b/dolphinscheduler-service/pom.xml
@@ -0,0 +1,56 @@
+
+
+
+
+ dolphinscheduler
+ org.apache.dolphinscheduler
+ 1.2.1-SNAPSHOT
+
+ 4.0.0
+
+ dolphinscheduler-service
+
+ dolphinscheduler-service
+
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-remote
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-dao
+
+
+ org.apache.curator
+ curator-client
+ ${curator.version}
+
+
+ log4j-1.2-api
+ org.apache.logging.log4j
+
+
+ io.netty
+ netty
+
+
+
+
+ org.quartz-scheduler
+ quartz
+
+
+ c3p0
+ c3p0
+
+
+
+
+
+ org.quartz-scheduler
+ quartz-jobs
+
+
+
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java
similarity index 96%
rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java
index 97618e1b39..ddf1fecf76 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java
@@ -14,14 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.utils;
+package org.apache.dolphinscheduler.service.bean;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
-
@Component
public class SpringApplicationContext implements ApplicationContextAware {
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java
new file mode 100644
index 0000000000..5daf535625
--- /dev/null
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.service.log;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.remote.NettyRemotingClient;
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.log.*;
+import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
+import org.apache.dolphinscheduler.remote.utils.Address;
+import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * log client
+ */
+public class LogClientService {
+
+ private static final Logger logger = LoggerFactory.getLogger(LogClientService.class);
+
+ private final NettyClientConfig clientConfig;
+
+ private final NettyRemotingClient client;
+
+ /**
+ * request time out
+ */
+ private final long logRequestTimeout = 10 * 1000;
+
+ /**
+ * construct client
+ */
+ public LogClientService() {
+ this.clientConfig = new NettyClientConfig();
+ this.clientConfig.setWorkerThreads(4);
+ this.client = new NettyRemotingClient(clientConfig);
+ }
+
+ /**
+ * close
+ */
+ public void close() {
+ this.client.close();
+ logger.info("logger client closed");
+ }
+
+ /**
+ * roll view log
+ * @param host host
+ * @param port port
+ * @param path path
+ * @param skipLineNum skip line number
+ * @param limit limit
+ * @return log content
+ */
+ public String rollViewLog(String host, int port, String path,int skipLineNum,int limit) {
+ logger.info("roll view log, host : {}, port : {}, path {}, skipLineNum {} ,limit {}", host, port, path, skipLineNum, limit);
+ RollViewLogRequestCommand request = new RollViewLogRequestCommand(path, skipLineNum, limit);
+ String result = "";
+ final Address address = new Address(host, port);
+ try {
+ Command command = request.convert2Command();
+ Command response = this.client.sendSync(address, command, logRequestTimeout);
+ if(response != null){
+ RollViewLogResponseCommand rollReviewLog = FastJsonSerializer.deserialize(
+ response.getBody(), RollViewLogResponseCommand.class);
+ return rollReviewLog.getMsg();
+ }
+ } catch (Exception e) {
+ logger.error("roll view log error", e);
+ } finally {
+ this.client.closeChannel(address);
+ }
+ return result;
+ }
+
+ /**
+ * view log
+ * @param host host
+ * @param port port
+ * @param path path
+ * @return log content
+ */
+ public String viewLog(String host, int port, String path) {
+ logger.info("view log path {}", path);
+ ViewLogRequestCommand request = new ViewLogRequestCommand(path);
+ String result = "";
+ final Address address = new Address(host, port);
+ try {
+ Command command = request.convert2Command();
+ Command response = this.client.sendSync(address, command, logRequestTimeout);
+ if(response != null){
+ ViewLogResponseCommand viewLog = FastJsonSerializer.deserialize(
+ response.getBody(), ViewLogResponseCommand.class);
+ return viewLog.getMsg();
+ }
+ } catch (Exception e) {
+ logger.error("view log error", e);
+ } finally {
+ this.client.closeChannel(address);
+ }
+ return result;
+ }
+
+ /**
+ * get log size
+ * @param host host
+ * @param port port
+ * @param path log path
+ * @return log content bytes
+ */
+ public byte[] getLogBytes(String host, int port, String path) {
+ logger.info("log path {}", path);
+ GetLogBytesRequestCommand request = new GetLogBytesRequestCommand(path);
+ byte[] result = null;
+ final Address address = new Address(host, port);
+ try {
+ Command command = request.convert2Command();
+ Command response = this.client.sendSync(address, command, logRequestTimeout);
+ if(response != null){
+ GetLogBytesResponseCommand getLog = FastJsonSerializer.deserialize(
+ response.getBody(), GetLogBytesResponseCommand.class);
+ return getLog.getData();
+ }
+ } catch (Exception e) {
+ logger.error("get log size error", e);
+ } finally {
+ this.client.closeChannel(address);
+ }
+ return result;
+ }
+}
\ No newline at end of file
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java
new file mode 100644
index 0000000000..98ee3fdbbf
--- /dev/null
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.service.log;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * log asyc callback
+ */
+public class LogPromise {
+
+ private static final ConcurrentHashMap PROMISES = new ConcurrentHashMap<>();
+
+ /**
+ * request unique identification
+ */
+ private long opaque;
+
+ /**
+ * start timemillis
+ */
+ private final long start;
+
+ /**
+ * timeout
+ */
+ private final long timeout;
+
+ /**
+ * latch
+ */
+ private final CountDownLatch latch;
+
+ /**
+ * result
+ */
+ private Object result;
+
+ public LogPromise(long opaque, long timeout){
+ this.opaque = opaque;
+ this.timeout = timeout;
+ this.start = System.currentTimeMillis();
+ this.latch = new CountDownLatch(1);
+ PROMISES.put(opaque, this);
+ }
+
+
+ /**
+ * notify client finish
+ * @param opaque unique identification
+ * @param result result
+ */
+ public static void notify(long opaque, Object result){
+ LogPromise promise = PROMISES.remove(opaque);
+ if(promise != null){
+ promise.doCountDown(result);
+ }
+ }
+
+ /**
+ * countdown
+ *
+ * @param result result
+ */
+ private void doCountDown(Object result){
+ this.result = result;
+ this.latch.countDown();
+ }
+
+ /**
+ * whether timeout
+ * @return timeout
+ */
+ public boolean isTimeout(){
+ return System.currentTimeMillis() - start > timeout;
+ }
+
+ /**
+ * get result
+ * @return
+ */
+ public Object getResult(){
+ try {
+ latch.await(timeout, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException ignore) {
+ }
+ PROMISES.remove(opaque);
+ return this.result;
+ }
+
+
+}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/permission/PermissionCheck.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java
similarity index 80%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/permission/PermissionCheck.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java
index 63d4c1c8af..027666f053 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/permission/PermissionCheck.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java
@@ -14,13 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.permission;
+package org.apache.dolphinscheduler.service.permission;
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import java.util.List;
@@ -38,7 +38,7 @@ public class PermissionCheck {
/**
* Authorization Type
*/
- private ProcessDao processDao;
+ private ProcessService processService;
/**
* need check array
@@ -53,23 +53,23 @@ public class PermissionCheck {
/**
* permission check
* @param authorizationType authorization type
- * @param processDao process dao
+ * @param processService process dao
*/
- public PermissionCheck(AuthorizationType authorizationType, ProcessDao processDao) {
+ public PermissionCheck(AuthorizationType authorizationType, ProcessService processService) {
this.authorizationType = authorizationType;
- this.processDao = processDao;
+ this.processService = processService;
}
/**
* permission check
* @param authorizationType
- * @param processDao
+ * @param processService
* @param needChecks
* @param userId
*/
- public PermissionCheck(AuthorizationType authorizationType, ProcessDao processDao, T[] needChecks, int userId) {
+ public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId) {
this.authorizationType = authorizationType;
- this.processDao = processDao;
+ this.processService = processService;
this.needChecks = needChecks;
this.userId = userId;
}
@@ -77,14 +77,14 @@ public class PermissionCheck {
/**
* permission check
* @param authorizationType
- * @param processDao
+ * @param processService
* @param needChecks
* @param userId
* @param logger
*/
- public PermissionCheck(AuthorizationType authorizationType, ProcessDao processDao, T[] needChecks, int userId,Logger logger) {
+ public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId, Logger logger) {
this.authorizationType = authorizationType;
- this.processDao = processDao;
+ this.processService = processService;
this.needChecks = needChecks;
this.userId = userId;
this.logger = logger;
@@ -98,12 +98,12 @@ public class PermissionCheck {
this.authorizationType = authorizationType;
}
- public ProcessDao getProcessDao() {
- return processDao;
+ public ProcessService getProcessService() {
+ return processService;
}
- public void setProcessDao(ProcessDao processDao) {
- this.processDao = processDao;
+ public void setProcessService(ProcessService processService) {
+ this.processService = processService;
}
public T[] getNeedChecks() {
@@ -142,9 +142,9 @@ public class PermissionCheck {
public void checkPermission() throws Exception{
if(this.needChecks.length > 0){
// get user type in order to judge whether the user is admin
- User user = processDao.getUserById(userId);
+ User user = processService.getUserById(userId);
if (user.getUserType() != UserType.ADMIN_USER){
- List unauthorizedList = processDao.listUnauthorized(userId,needChecks,authorizationType);
+ List unauthorizedList = processService.listUnauthorized(userId,needChecks,authorizationType);
// if exist unauthorized resource
if(CollectionUtils.isNotEmpty(unauthorizedList)){
logger.error("user {} didn't has permission of {}: {}", user.getUserName(), authorizationType.getDescp(),unauthorizedList.toString());
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
similarity index 98%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
index 820b2fdaf4..b589cd4295 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao;
+package org.apache.dolphinscheduler.service.process;
import com.alibaba.fastjson.JSONObject;
import com.cronutils.model.Cron;
@@ -24,16 +24,12 @@ import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.model.DateInterval;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.process.Property;
-import org.apache.dolphinscheduler.common.queue.ITaskQueue;
import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters;
-import org.apache.dolphinscheduler.common.utils.DateUtils;
-import org.apache.dolphinscheduler.common.utils.IpUtils;
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.common.utils.ParameterUtils;
-import org.apache.dolphinscheduler.common.utils.StringUtils;
+import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.mapper.*;
-import org.apache.dolphinscheduler.dao.utils.cron.CronUtils;
+import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
+import org.apache.dolphinscheduler.service.queue.ITaskQueue;
import org.quartz.CronExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,7 +47,7 @@ import static org.apache.dolphinscheduler.common.Constants.*;
* process relative dao that some mappers in this.
*/
@Component
-public class ProcessDao {
+public class ProcessService {
private final Logger logger = LoggerFactory.getLogger(getClass());
@@ -481,7 +477,7 @@ public class ProcessDao {
if(cmdParam == null
|| !cmdParam.containsKey(Constants.CMDPARAM_START_NODE_NAMES)
|| cmdParam.get(Constants.CMDPARAM_START_NODE_NAMES).isEmpty()){
- logger.error(String.format("command node depend type is %s, but start nodes is null ", command.getTaskDependType().toString()));
+ logger.error("command node depend type is {}, but start nodes is null ", command.getTaskDependType());
return false;
}
}
@@ -504,7 +500,7 @@ public class ProcessDao {
if(command.getProcessDefinitionId() != 0){
processDefinition = processDefineMapper.selectById(command.getProcessDefinitionId());
if(processDefinition == null){
- logger.error(String.format("cannot find the work process define! define id : %d", command.getProcessDefinitionId()));
+ logger.error("cannot find the work process define! define id : {}", command.getProcessDefinitionId());
return null;
}
}
@@ -954,6 +950,7 @@ public class ProcessDao {
}
}
}
+ taskInstance.setExecutorId(processInstance.getExecutorId());
taskInstance.setProcessInstancePriority(processInstance.getProcessInstancePriority());
taskInstance.setState(getSubmitTaskState(taskInstance, processInstanceState));
taskInstance.setSubmitTime(new Date());
@@ -976,21 +973,21 @@ public class ProcessDao {
return true;
}
if(taskInstance.getState().typeIsFinished()){
- logger.info(String.format("submit to task queue, but task [%s] state [%s] is already finished. ", taskInstance.getName(), taskInstance.getState().toString()));
+ logger.info("submit to task queue, but task [{}] state [{}] is already finished. ", taskInstance.getName(), taskInstance.getState());
return true;
}
// task cannot submit when running
if(taskInstance.getState() == ExecutionStatus.RUNNING_EXEUTION){
- logger.info(String.format("submit to task queue, but task [%s] state already be running. ", taskInstance.getName()));
+ logger.info("submit to task queue, but task [{}] state already be running. ", taskInstance.getName());
return true;
}
if(checkTaskExistsInTaskQueue(taskInstance)){
- logger.info(String.format("submit to task queue, but task [%s] already exists in the queue.", taskInstance.getName()));
+ logger.info("submit to task queue, but task [{}] already exists in the queue.", taskInstance.getName());
return true;
}
logger.info("task ready to queue: {}" , taskInstance);
boolean insertQueueResult = taskQueue.add(DOLPHINSCHEDULER_TASKS_QUEUE, taskZkInfo(taskInstance));
- logger.info(String.format("master insert into queue success, task : %s", taskInstance.getName()) );
+ logger.info("master insert into queue success, task : {}", taskInstance.getName());
return insertQueueResult;
}catch (Exception e){
logger.error("submit task to queue Exception: ", e);
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/DruidConnectionProvider.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/DruidConnectionProvider.java
similarity index 99%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/DruidConnectionProvider.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/DruidConnectionProvider.java
index 8a4ceba927..d51e8e82bf 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/DruidConnectionProvider.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/DruidConnectionProvider.java
@@ -14,11 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.quartz;
+package org.apache.dolphinscheduler.service.quartz;
import com.alibaba.druid.pool.DruidDataSource;
import org.quartz.SchedulerException;
import org.quartz.utils.ConnectionProvider;
+
import java.sql.Connection;
import java.sql.SQLException;
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/ProcessScheduleJob.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java
similarity index 83%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/ProcessScheduleJob.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java
index ac461296a9..69a80e65f5 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/ProcessScheduleJob.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/ProcessScheduleJob.java
@@ -14,17 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.quartz;
+package org.apache.dolphinscheduler.service.quartz;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.ReleaseState;
-import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Schedule;
-import org.quartz.*;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.quartz.Job;
+import org.quartz.JobDataMap;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
@@ -42,17 +45,17 @@ public class ProcessScheduleJob implements Job {
private static final Logger logger = LoggerFactory.getLogger(ProcessScheduleJob.class);
/**
- * process dao
+ * process service
*/
- private static ProcessDao processDao;
+ private static ProcessService processService;
/**
* init
- * @param processDao process dao
+ * @param processService process dao
*/
- public static void init(ProcessDao processDao) {
- ProcessScheduleJob.processDao = processDao;
+ public static void init(ProcessService processService) {
+ ProcessScheduleJob.processService = processService;
}
/**
@@ -64,7 +67,7 @@ public class ProcessScheduleJob implements Job {
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
- Assert.notNull(processDao, "please call init() method first");
+ Assert.notNull(processService, "please call init() method first");
JobDataMap dataMap = context.getJobDetail().getJobDataMap();
@@ -80,7 +83,7 @@ public class ProcessScheduleJob implements Job {
logger.info("scheduled fire time :{}, fire time :{}, process id :{}", scheduledFireTime, fireTime, scheduleId);
// query schedule
- Schedule schedule = processDao.querySchedule(scheduleId);
+ Schedule schedule = processService.querySchedule(scheduleId);
if (schedule == null) {
logger.warn("process schedule does not exist in db，delete schedule job in quartz, projectId:{}, scheduleId:{}", projectId, scheduleId);
deleteJob(projectId, scheduleId);
@@ -88,7 +91,7 @@ public class ProcessScheduleJob implements Job {
}
- ProcessDefinition processDefinition = processDao.findProcessDefineById(schedule.getProcessDefinitionId());
+ ProcessDefinition processDefinition = processService.findProcessDefineById(schedule.getProcessDefinitionId());
// release state : online/offline
ReleaseState releaseState = processDefinition.getReleaseState();
if (processDefinition == null || releaseState == ReleaseState.OFFLINE) {
@@ -108,7 +111,7 @@ public class ProcessScheduleJob implements Job {
command.setWarningType(schedule.getWarningType());
command.setProcessInstancePriority(schedule.getProcessInstancePriority());
- processDao.createCommand(command);
+ processService.createCommand(command);
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/QuartzExecutors.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java
similarity index 97%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/QuartzExecutors.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java
index 054d7903fc..60cdb1dd97 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/quartz/QuartzExecutors.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java
@@ -14,12 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.quartz;
+package org.apache.dolphinscheduler.service.quartz;
+import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.Schedule;
-import org.apache.commons.lang.StringUtils;
import org.quartz.*;
import org.quartz.impl.StdSchedulerFactory;
import org.quartz.impl.matchers.GroupMatcher;
@@ -223,7 +223,7 @@ public class QuartzExecutors {
}
} catch (SchedulerException e) {
- logger.error(String.format("delete job : %s failed",jobName), e);
+ logger.error("delete job : {} failed",jobName, e);
} finally {
lock.writeLock().unlock();
}
@@ -247,7 +247,7 @@ public class QuartzExecutors {
return scheduler.deleteJobs(jobKeys);
} catch (SchedulerException e) {
- logger.error(String.format("delete all jobs in job group: %s failed",jobGroupName), e);
+ logger.error("delete all jobs in job group: {} failed",jobGroupName, e);
} finally {
lock.writeLock().unlock();
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/AbstractCycle.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java
similarity index 99%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/AbstractCycle.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java
index 0cda336d7d..0a2e31b610 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/AbstractCycle.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java
@@ -14,13 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.utils.cron;
+package org.apache.dolphinscheduler.service.quartz.cron;
-import org.apache.dolphinscheduler.common.enums.CycleEnum;
import com.cronutils.model.Cron;
import com.cronutils.model.field.CronField;
import com.cronutils.model.field.CronFieldName;
import com.cronutils.model.field.expression.*;
+import org.apache.dolphinscheduler.common.enums.CycleEnum;
/**
* Cycle
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java
similarity index 95%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java
index 8a9087a33c..d03a4a5cdc 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java
@@ -14,15 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.utils.cron;
+package org.apache.dolphinscheduler.service.quartz.cron;
-import org.apache.dolphinscheduler.common.enums.CycleEnum;
-import org.apache.dolphinscheduler.common.thread.Stopper;
-import org.apache.dolphinscheduler.common.utils.DateUtils;
import com.cronutils.model.Cron;
import com.cronutils.model.definition.CronDefinitionBuilder;
import com.cronutils.parser.CronParser;
+import org.apache.dolphinscheduler.common.enums.CycleEnum;
+import org.apache.dolphinscheduler.common.thread.Stopper;
+import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.quartz.CronExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,14 +31,16 @@ import java.text.ParseException;
import java.util.*;
import static com.cronutils.model.CronType.QUARTZ;
-import static org.apache.dolphinscheduler.dao.utils.cron.CycleFactory.*;
+import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.*;
/**
* cron utils
*/
public class CronUtils {
-
+ private CronUtils() {
+ throw new IllegalStateException("CronUtils class");
+ }
private static final Logger logger = LoggerFactory.getLogger(CronUtils.class);
@@ -169,7 +171,7 @@ public class CronUtils {
cronExpression = parse2CronExpression(cron);
}catch (ParseException e){
logger.error(e.getMessage(), e);
- return Collections.EMPTY_LIST;
+ return Collections.emptyList();
}
return getSelfFireDateList(startTime, endTime, cronExpression);
}
@@ -202,7 +204,7 @@ public class CronUtils {
calendar.add(Calendar.DATE, 1);
break;
default:
- logger.error("Dependent process definition's cycleEnum is {},not support!!", cycleEnum.name());
+ logger.error("Dependent process definition's cycleEnum is {},not support!!", cycleEnum);
break;
}
maxExpirationTime = calendar.getTime();
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleFactory.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleFactory.java
similarity index 97%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleFactory.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleFactory.java
index 10906b42a3..1f807dce7f 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleFactory.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleFactory.java
@@ -14,18 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.utils.cron;
+package org.apache.dolphinscheduler.service.quartz.cron;
-import org.apache.dolphinscheduler.common.enums.CycleEnum;
import com.cronutils.model.Cron;
import com.cronutils.model.field.expression.Always;
import com.cronutils.model.field.expression.QuestionMark;
+import org.apache.dolphinscheduler.common.enums.CycleEnum;
/**
* Crontab Cycle Tool Factory
*/
public class CycleFactory {
-
+ private CycleFactory() {
+ throw new IllegalStateException("CycleFactory class");
+ }
/**
* min
* @param cron cron
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleLinks.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleLinks.java
similarity index 97%
rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleLinks.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleLinks.java
index 63824bda8e..9f01b18868 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CycleLinks.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleLinks.java
@@ -14,10 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.utils.cron;
+package org.apache.dolphinscheduler.service.quartz.cron;
-import org.apache.dolphinscheduler.common.enums.CycleEnum;
import com.cronutils.model.Cron;
+import org.apache.dolphinscheduler.common.enums.CycleEnum;
import java.util.ArrayList;
import java.util.List;
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/ITaskQueue.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/ITaskQueue.java
similarity index 97%
rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/ITaskQueue.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/ITaskQueue.java
index 5beb8111ad..bed8a11247 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/ITaskQueue.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/ITaskQueue.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.queue;
+package org.apache.dolphinscheduler.service.queue;
import java.util.List;
import java.util.Set;
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueFactory.java
similarity index 93%
rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueFactory.java
index 0a2d943118..6be419f5a9 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueFactory.java
@@ -14,11 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.queue;
+package org.apache.dolphinscheduler.service.queue;
-import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.dolphinscheduler.common.utils.SpringApplicationContext;
+import org.apache.dolphinscheduler.common.utils.CommonUtils;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueZkImpl.java
similarity index 97%
rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueZkImpl.java
index d442c13ebc..9c1d318ea5 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskQueueZkImpl.java
@@ -14,13 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.queue;
+package org.apache.dolphinscheduler.service.queue;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.IpUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
-import org.apache.dolphinscheduler.common.zk.ZookeeperOperator;
+import org.apache.dolphinscheduler.service.zk.ZookeeperOperator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -269,7 +269,7 @@ public class TaskQueueZkImpl implements ITaskQueue {
zookeeperOperator.remove(taskIdPath);
}catch(Exception e){
- logger.error(String.format("delete task:%s from zookeeper fail, exception:" ,nodeValue) ,e);
+ logger.error("delete task:{} from zookeeper fail, exception:" ,nodeValue ,e);
}
}
@@ -318,7 +318,7 @@ public class TaskQueueZkImpl implements ITaskQueue {
zookeeperOperator.remove(path + value);
}catch(Exception e){
- logger.error(String.format("delete task:" + value + " exception"),e);
+ logger.error("delete task:{} exception",value,e);
}
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java
similarity index 99%
rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java
index f62e106680..135bfdabc6 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.zk;
+package org.apache.dolphinscheduler.service.zk;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.imps.CuratorFrameworkState;
@@ -31,12 +31,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
+
import static org.apache.dolphinscheduler.common.Constants.*;
/**
* abstract zookeeper client
*/
-public abstract class AbstractZKClient extends ZookeeperCachedOperator{
+public abstract class AbstractZKClient extends ZookeeperCachedOperator {
private static final Logger logger = LoggerFactory.getLogger(AbstractZKClient.class);
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/DefaultEnsembleProvider.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java
similarity index 96%
rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/DefaultEnsembleProvider.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java
index 0cf06c0503..9eedf7a4ca 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/DefaultEnsembleProvider.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.zk;
+package org.apache.dolphinscheduler.service.zk;
import org.apache.curator.ensemble.EnsembleProvider;
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java
similarity index 90%
rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java
index 5aa25552d7..dccb768f8b 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java
@@ -14,22 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.zk;
+package org.apache.dolphinscheduler.service.zk;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.TreeCache;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
-import org.apache.curator.framework.recipes.cache.TreeCacheListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.nio.charset.StandardCharsets;
-import java.util.concurrent.ConcurrentHashMap;
-
-import static org.apache.dolphinscheduler.common.utils.Preconditions.*;
-import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull;
@Component
public class ZookeeperCachedOperator extends ZookeeperOperator {
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperConfig.java
similarity index 98%
rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperConfig.java
index 75a9f6c5f4..c6bdfc3b02 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperConfig.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.zk;
+package org.apache.dolphinscheduler.service.zk;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.PropertySource;
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java
similarity index 98%
rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java
index 9442afd7a0..a2cabce805 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.zk;
+package org.apache.dolphinscheduler.service.zk;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
@@ -33,12 +33,10 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
import java.nio.charset.StandardCharsets;
import java.util.List;
-import static org.apache.dolphinscheduler.common.utils.Preconditions.*;
import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull;
/**
diff --git a/dolphinscheduler-common/src/main/resources/quartz.properties b/dolphinscheduler-service/src/main/resources/quartz.properties
similarity index 96%
rename from dolphinscheduler-common/src/main/resources/quartz.properties
rename to dolphinscheduler-service/src/main/resources/quartz.properties
index 2e3a2a0dc1..9c8930b647 100644
--- a/dolphinscheduler-common/src/main/resources/quartz.properties
+++ b/dolphinscheduler-service/src/main/resources/quartz.properties
@@ -59,6 +59,6 @@ org.quartz.jobStore.dataSource = myDs
#============================================================================
# Configure Datasources
#============================================================================
-org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.dao.quartz.DruidConnectionProvider
+org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider
org.quartz.dataSource.myDs.maxConnections = 10
org.quartz.dataSource.myDs.validationQuery = select 1
\ No newline at end of file
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java b/dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java
similarity index 90%
rename from dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java
rename to dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java
index 1135cf20f5..6a402b5e67 100644
--- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java
+++ b/dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java
@@ -14,11 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.dao.cron;
+package cron;
-import org.apache.dolphinscheduler.common.enums.CycleEnum;
-import org.apache.dolphinscheduler.common.utils.DateUtils;
-import org.apache.dolphinscheduler.dao.utils.cron.CronUtils;
import com.cronutils.builder.CronBuilder;
import com.cronutils.model.Cron;
import com.cronutils.model.CronType;
@@ -26,6 +23,9 @@ import com.cronutils.model.definition.CronDefinitionBuilder;
import com.cronutils.model.field.CronField;
import com.cronutils.model.field.CronFieldName;
import com.cronutils.model.field.expression.*;
+import org.apache.dolphinscheduler.common.enums.CycleEnum;
+import org.apache.dolphinscheduler.common.utils.DateUtils;
+import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -61,7 +61,7 @@ public class CronUtilsTest {
String cronAsString = cron.asString();
// 0 */5 * * * ? * Every five minutes(once every 5 minutes)
- Assert.assertEquals(cronAsString, "0 */5 * * * ? *");
+ Assert.assertEquals("0 */5 * * * ? *", cronAsString);
}
@@ -74,12 +74,12 @@ public class CronUtilsTest {
String strCrontab = "0 1 2 3 * ? *";
Cron depCron = CronUtils.parse2Cron(strCrontab);
- Assert.assertEquals(depCron.retrieve(CronFieldName.SECOND).getExpression().asString(), "0");
- Assert.assertEquals(depCron.retrieve(CronFieldName.MINUTE).getExpression().asString(), "1");
- Assert.assertEquals(depCron.retrieve(CronFieldName.HOUR).getExpression().asString(), "2");
- Assert.assertEquals(depCron.retrieve(CronFieldName.DAY_OF_MONTH).getExpression().asString(), "3");
- Assert.assertEquals(depCron.retrieve(CronFieldName.MONTH).getExpression().asString(), "*");
- Assert.assertEquals(depCron.retrieve(CronFieldName.YEAR).getExpression().asString(), "*");
+ Assert.assertEquals("0", depCron.retrieve(CronFieldName.SECOND).getExpression().asString());
+ Assert.assertEquals("1", depCron.retrieve(CronFieldName.MINUTE).getExpression().asString());
+ Assert.assertEquals("2", depCron.retrieve(CronFieldName.HOUR).getExpression().asString());
+ Assert.assertEquals("3", depCron.retrieve(CronFieldName.DAY_OF_MONTH).getExpression().asString());
+ Assert.assertEquals("*", depCron.retrieve(CronFieldName.MONTH).getExpression().asString());
+ Assert.assertEquals("*", depCron.retrieve(CronFieldName.YEAR).getExpression().asString());
}
/**
@@ -89,13 +89,13 @@ public class CronUtilsTest {
@Test
public void testScheduleType() throws ParseException {
CycleEnum cycleEnum = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 */1 * * * ? *"));
- Assert.assertEquals(cycleEnum.name(), "MINUTE");
+ Assert.assertEquals("MINUTE", cycleEnum.name());
CycleEnum cycleEnum2 = CronUtils.getMaxCycle("0 * * * * ? *");
- Assert.assertEquals(cycleEnum2.name(), "MINUTE");
+ Assert.assertEquals("MINUTE", cycleEnum2.name());
CycleEnum cycleEnum3 = CronUtils.getMiniCycle(CronUtils.parse2Cron("0 * * * * ? *"));
- Assert.assertEquals(cycleEnum3.name(), "MINUTE");
+ Assert.assertEquals("MINUTE", cycleEnum3.name());
}
/**
@@ -164,6 +164,7 @@ public class CronUtilsTest {
logger.info("can't get scheduleType");
}
}
+ Assert.assertTrue(true);
}
@Test
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/BaseTaskQueueTest.java b/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java
similarity index 90%
rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/BaseTaskQueueTest.java
rename to dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java
index 433e4fa30f..a0cc457e22 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/BaseTaskQueueTest.java
+++ b/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java
@@ -14,9 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.queue;
+package queue;
-import org.apache.dolphinscheduler.common.zk.ZKServer;
+import org.apache.dolphinscheduler.service.queue.ITaskQueue;
+import org.apache.dolphinscheduler.service.queue.TaskQueueFactory;
import org.junit.*;
/**
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java b/dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java
similarity index 99%
rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java
rename to dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java
index b34a7d6924..d29c5aa610 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java
+++ b/dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.queue;
+package queue;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.IpUtils;
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/ZKServer.java b/dolphinscheduler-service/src/test/java/queue/ZKServer.java
similarity index 99%
rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/ZKServer.java
rename to dolphinscheduler-service/src/test/java/queue/ZKServer.java
index fc39e62ed8..65fb95c02b 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/ZKServer.java
+++ b/dolphinscheduler-service/src/test/java/queue/ZKServer.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.zk;
+package queue;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java b/dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java
similarity index 97%
rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java
rename to dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java
index dcb0e1370e..a1b85f1b12 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java
+++ b/dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java
@@ -14,8 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.utils;
+package utils;
+import org.apache.dolphinscheduler.common.utils.Preconditions;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -35,7 +36,7 @@ public class PreconditionsTest {
public void testCheckNotNull() throws Exception {
String testReference = "test reference";
//test reference is not null
- Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference));
+ Assert.assertEquals(testReference, Preconditions.checkNotNull(testReference));
Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"reference is null"));
Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"%s is null",testReference));
diff --git a/dolphinscheduler-ui/.env b/dolphinscheduler-ui/.env
index 4c7e96e795..e676be6059 100644
--- a/dolphinscheduler-ui/.env
+++ b/dolphinscheduler-ui/.env
@@ -17,4 +17,4 @@
API_BASE = http://192.168.xx.xx:12345
# If IP access is required for local development, remove the "#"
-#DEV_HOST = 192.168.xx.xx
\ No newline at end of file
+#DEV_HOST = 192.168.xx.xx
diff --git a/dolphinscheduler-ui/package.json b/dolphinscheduler-ui/package.json
index 421fd394d6..da15b722fc 100644
--- a/dolphinscheduler-ui/package.json
+++ b/dolphinscheduler-ui/package.json
@@ -53,7 +53,7 @@
"html-loader": "^0.5.5",
"html-webpack-plugin": "^3.2.0",
"mini-css-extract-plugin": "^0.8.2",
- "node-sass": "^4.13.0",
+ "node-sass": "^4.13.1",
"postcss-loader": "^3.0.0",
"progress-bar-webpack-plugin": "^1.12.1",
"rimraf": "^2.6.2",
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js
old mode 100644
new mode 100755
index e8187043bf..a9a51aa2b1
--- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js
@@ -283,6 +283,14 @@ let tasksType = {
'DATAX': {
desc: 'DataX',
color: '#1fc747'
+ },
+ 'SQOOP': {
+ desc: 'SQOOP',
+ color: '#E46F13'
+ },
+ 'CONDITIONS': {
+ desc: 'CONDITIONS',
+ color: '#E46F13'
}
}
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js
index 8225673ecd..240f3246aa 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js
@@ -49,10 +49,16 @@ Dag.prototype.setConfig = function (o) {
* create dag
*/
Dag.prototype.create = function () {
+ let self = this
jsPlumb.ready(() => {
JSP.init({
dag: this.dag,
- instance: this.instance
+ instance: this.instance,
+ options: {
+ onRemoveNodes ($id) {
+ self.dag.removeEventModelById($id)
+ }
+ }
})
// init event
@@ -108,7 +114,7 @@ Dag.prototype.backfill = function (arg) {
tmp.push(locationsValue2[i])
}
}
-
+
function copy (array) {
let newArray = []
for(let item of array) {
@@ -117,7 +123,7 @@ Dag.prototype.backfill = function (arg) {
return newArray;
}
-
+
let newArr = copy(arr)
function getNewArr() {
for(let i= 0; i1) {
dataObject[Object.keys(locationsValue1)[0]].y = (countTree/2)*120+50
}
-
+
locationsValue = dataObject
+ let self = this
jsPlumb.ready(() => {
JSP.init({
dag: this.dag,
- instance: this.instance
+ instance: this.instance,
+ options: {
+ onRemoveNodes ($id) {
+ self.dag.removeEventModelById($id)
+ }
+ }
})
// Backfill
JSP.jspBackfill({
@@ -298,10 +310,16 @@ Dag.prototype.backfill = function (arg) {
})
})
} else {
+ let self = this
jsPlumb.ready(() => {
JSP.init({
dag: this.dag,
- instance: this.instance
+ instance: this.instance,
+ options: {
+ onRemoveNodes ($id) {
+ self.dag.removeEventModelById($id)
+ }
+ }
})
// Backfill
JSP.jspBackfill({
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss
old mode 100644
new mode 100755
index 420bae8c89..6d97856960
--- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss
@@ -104,6 +104,12 @@
.icos-DATAX {
background: url("../img/toolbar_DATAX.png") no-repeat 50% 50%;
}
+ .icos-SQOOP {
+ background: url("../img/toolbar_SQOOP.png") no-repeat 50% 50%;
+ }
+ .icos-CONDITIONS {
+ background: url("../img/toobar_CONDITIONS.png") no-repeat 50% 50%;
+ }
.toolbar {
width: 60px;
height: 100%;
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue
old mode 100644
new mode 100755
index 40b6d85198..d912a9a884
--- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue
@@ -61,7 +61,7 @@
-
+
-
0) {
+ rearNode.forEach(v => {
+ let rearobj = {}
+ rearobj.value = $(`#${v}`).find('.name-p').text()
+ rearobj.label = $(`#${v}`).find('.name-p').text()
+ rearList.push(rearobj)
+ })
+ } else {
+ rearList = []
+ }
+ let targetarr = $(`#${id}`).attr('data-targetarr')
+ if (targetarr) {
+ let nodearr = targetarr.split(',')
+ nodearr.forEach(v => {
+ let nodeobj = {}
+ nodeobj.value = $(`#${v}`).find('.name-p').text()
+ nodeobj.label = $(`#${v}`).find('.name-p').text()
+ preNode.push(nodeobj)
+ })
+ } else {
+ preNode = []
+ }
if (eventModel) {
eventModel.remove()
}
@@ -486,6 +514,7 @@
}
this.taskId = id
+ type = type || self.dagBarId
eventModel = this.$drawer({
closable: false,
@@ -522,11 +551,18 @@
},
props: {
id: id,
- taskType: type || self.dagBarId,
- self: self
+ taskType: type,
+ self: self,
+ preNode: preNode,
+ rearList: rearList
}
})
})
+ },
+ removeEventModelById ($id) {
+ if(eventModel && this.taskId == $id){
+ eventModel.remove()
+ }
}
},
watch: {
@@ -580,6 +616,9 @@
clearInterval(this.setIntervalP)
},
destroyed () {
+ if (eventModel) {
+ eventModel.remove()
+ }
},
computed: {
...mapState('dag', ['tasks', 'locations', 'connects', 'isEditDag', 'name'])
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue
old mode 100644
new mode 100755
index e05de8e880..2500ce5772
--- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue
@@ -109,6 +109,43 @@
({{$t('Minute')}})
+
+ {{$t('State')}}
+
+
+
+
+
+
+
+
+ {{$t('Branch flow')}}
+
+
+
+
+
+
+
+ {{$t('State')}}
+
+
+
+
+
+
+
+
+ {{$t('Branch flow')}}
+
+
+
+
+
+
@@ -229,6 +280,8 @@
import mDependent from './tasks/dependent'
import mHttp from './tasks/http'
import mDatax from './tasks/datax'
+ import mConditions from './tasks/CONDITIONS'
+ import mSqoop from './tasks/sqoop'
import mSubProcess from './tasks/sub_process'
import mSelectInput from './_source/selectInput'
import mTimeoutAlarm from './_source/timeoutAlarm'
@@ -245,13 +298,21 @@
// loading
spinnerLoading: false,
// node name
- name: ``,
+ name: '',
// description
description: '',
// Node echo data
backfillItem: {},
// Resource(list)
resourcesList: [],
+ successNode: 'success',
+ failedNode: 'failed',
+ successBranch: '',
+ failedBranch: '',
+ conditionResult: {
+ 'successNode': [],
+ 'failedNode': []
+ },
// dependence
dependence: {},
// cache dependence
@@ -271,7 +332,17 @@
// Task priority
taskInstancePriority: 'MEDIUM',
// worker group id
- workerGroupId: -1
+ workerGroupId: -1,
+ stateList:[
+ {
+ value: 'success',
+ label: `${i18n.$t('success')}`
+ },
+ {
+ value: 'failed',
+ label: `${i18n.$t('failed')}`
+ }
+ ]
}
},
/**
@@ -282,7 +353,9 @@
props: {
id: Number,
taskType: String,
- self: Object
+ self: Object,
+ preNode: Array,
+ rearList: Array
},
methods: {
/**
@@ -391,6 +464,10 @@
this.$message.warning(`${i18n.$t('Please enter name (required)')}`)
return false
}
+ if (this.successBranch !='' && this.successBranch == this.failedBranch) {
+ this.$message.warning(`${i18n.$t('Cannot select the same node for successful branch flow and failed branch flow')}`)
+ return false
+ }
if (this.name === this.backfillItem.name) {
return true
}
@@ -419,6 +496,8 @@
}
$(`#${this.id}`).find('span').text(this.name)
+ this.conditionResult.successNode[0] = this.successBranch
+ this.conditionResult.failedNode[0] = this.failedBranch
// Store the corresponding node data structure
this.$emit('addTaskInfo', {
item: {
@@ -428,12 +507,15 @@
params: this.params,
description: this.description,
runFlag: this.runFlag,
+ conditionResult: this.conditionResult,
dependence: this.dependence,
maxRetryTimes: this.maxRetryTimes,
retryInterval: this.retryInterval,
timeout: this.timeout,
taskInstancePriority: this.taskInstancePriority,
- workerGroupId: this.workerGroupId
+ workerGroupId: this.workerGroupId,
+ status: this.status,
+ branch: this.branch
},
fromThis: this
})
@@ -518,7 +600,10 @@
this.description = o.description
this.maxRetryTimes = o.maxRetryTimes
this.retryInterval = o.retryInterval
-
+ if(o.conditionResult) {
+ this.successBranch = o.conditionResult.successNode[0]
+ this.failedBranch = o.conditionResult.failedNode[0]
+ }
// If the workergroup has been deleted, set the default workergroup
var hasMatch = false;
for (let i = 0; i < this.store.state.security.workerGroupsListAll.length; i++) {
@@ -589,6 +674,8 @@
mDependent,
mHttp,
mDatax,
+ mSqoop,
+ mConditions,
mSelectInput,
mTimeoutAlarm,
mPriority,
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js
old mode 100644
new mode 100755
index fc8fe654d2..cdf632f13d
--- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js
@@ -232,6 +232,16 @@ const positionList = [
code: "Headers"
}
]
+const nodeStatusList = [
+ {
+ value: 'SUCCESS',
+ label: `${i18n.$t('success')}`
+ },
+ {
+ value: 'FAILURE',
+ label: `${i18n.$t('failed')}`
+ }
+]
export {
cycleList,
@@ -239,5 +249,6 @@ export {
typeList,
directList,
sqlTypeList,
- positionList
+ positionList,
+ nodeStatusList
}
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue
new file mode 100644
index 0000000000..4afb8b46c5
--- /dev/null
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue
new file mode 100644
index 0000000000..4ac04d91a6
--- /dev/null
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+
+ {{$t('Custom Parameters')}}
+
+
+
+
+ {{relation === 'AND' ? $t('and') : $t('or')}}
+
+
+
+ {{el.relation === 'AND' ? $t('and') : $t('or')}}
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue
new file mode 100644
index 0000000000..eead745a06
--- /dev/null
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue
@@ -0,0 +1,981 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+
+
+ {{$t('Direct')}}
+
+
+
+
+
+
+
+
+
+
+ {{$t('Data Source')}}
+
+
+
+ {{$t('Type')}}
+
+
+
+
+
+
+
+
+
+
+
+ {{$t('Datasource')}}
+
+
+
+
+
+
+
+ {{$t('ModelType')}}
+
+
+ {{$t('Form')}}
+ SQL
+
+
+
+
+
+
+
+ {{$t('Table')}}
+
+
+
+
+
+
+
+ {{$t('ColumnType')}}
+
+
+ {{$t('All Columns')}}
+ {{$t('Some Columns')}}
+
+
+
+
+
+ {{$t('Column')}}
+
+
+
+
+
+
+
+
+
+
+
+ {{$t('Database')}}
+
+
+
+
+
+
+ {{$t('Table')}}
+
+
+
+
+
+
+ {{$t('Hive partition Keys')}}
+
+
+
+
+
+
+ {{$t('Hive partition Values')}}
+
+
+
+
+
+
+
+
+
+ {{$t('Export Dir')}}
+
+
+
+
+
+
+
+
+ {{$t('SQL Statement')}}
+
+
+
+
+
+ {{$t('Map Column Hive')}}
+
+
+
+
+
+
+ {{$t('Map Column Java')}}
+
+
+
+
+
+
+
+
+ {{$t('Data Target')}}
+
+
+
+
+ {{$t('Type')}}
+
+
+
+
+
+
+
+
+
+ {{$t('Database')}}
+
+
+
+
+
+
+ {{$t('Table')}}
+
+
+
+
+
+
+ {{$t('CreateHiveTable')}}
+
+
+
+
+
+ {{$t('DropDelimiter')}}
+
+
+
+
+
+ {{$t('OverWriteSrc')}}
+
+
+
+
+
+ {{$t('ReplaceDelimiter')}}
+
+
+
+
+
+
+ {{$t('Hive partition Keys')}}
+
+
+
+
+
+
+ {{$t('Hive partition Values')}}
+
+
+
+
+
+
+
+
+ {{$t('Target Dir')}}
+
+
+
+
+
+
+ {{$t('DeleteTargetDir')}}
+
+
+
+
+
+ {{$t('CompressionCodec')}}
+
+
+ snappy
+ lzo
+ gzip
+ no
+
+
+
+
+ {{$t('FileType')}}
+
+
+ avro
+ sequence
+ text
+ parquet
+
+
+
+
+ {{$t('FieldsTerminated')}}
+
+
+
+
+
+
+ {{$t('LinesTerminated')}}
+
+
+
+
+
+
+
+
+ {{$t('Datasource')}}
+
+
+
+
+
+
+ {{$t('Table')}}
+
+
+
+
+
+
+ {{$t('Column')}}
+
+
+
+
+
+
+ {{$t('FieldsTerminated')}}
+
+
+
+
+
+
+ {{$t('LinesTerminated')}}
+
+
+
+
+
+
+ {{$t('IsUpdate')}}
+
+
+
+
+
+ {{$t('UpdateKey')}}
+
+
+
+
+
+
+ {{$t('UpdateMode')}}
+
+
+ {{$t('OnlyUpdate')}}
+ {{$t('AllowInsert')}}
+
+
+
+
+
+
+ {{$t('Concurrency')}}
+
+
+
+
+
+
+
+
+ {{$t('Custom Parameters')}}
+
+
+
+
+
+
+
+
+
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js
old mode 100644
new mode 100755
index 6523a1c298..598c94209e
--- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js
@@ -58,11 +58,13 @@ let JSP = function () {
/**
* dag init
*/
-JSP.prototype.init = function ({ dag, instance }) {
+JSP.prototype.init = function ({ dag, instance, options }) {
// Get the dag component instance
this.dag = dag
// Get jsplumb instance
this.JspInstance = instance
+ // Get JSP options
+ this.options = options || {}
// Register jsplumb connection type and configuration
this.JspInstance.registerConnectionType('basic', {
anchor: 'Continuous',
@@ -133,15 +135,6 @@ JSP.prototype.draggable = function () {
helper: 'clone',
containment: $('.dag-model'),
stop: function (e, ui) {
- self.tasksEvent(selfId)
-
- // Dom structure is not generated without pop-up form form
- if ($(`#${selfId}`).html()) {
- // dag event
- findComponentDownward(self.dag.$root, 'dag-chart')._createNodes({
- id: selfId
- })
- }
},
drag: function () {
$('body').find('.tooltip.fade.top.in').remove()
@@ -176,6 +169,16 @@ JSP.prototype.draggable = function () {
self.initNode(thisDom[thisDom.length - 1])
})
selfId = id
+
+ self.tasksEvent(selfId)
+
+ // Dom structure is not generated without pop-up form form
+ if ($(`#${selfId}`).html()) {
+ // dag event
+ findComponentDownward(self.dag.$root, 'dag-chart')._createNodes({
+ id: selfId
+ })
+ }
}
})
}
@@ -195,7 +198,8 @@ JSP.prototype.jsonHandle = function ({ largeJson, locations }) {
targetarr: locations[v.id]['targetarr'],
isAttachment: this.config.isAttachment,
taskType: v.type,
- runFlag: v.runFlag
+ runFlag: v.runFlag,
+ nodenumber: locations[v.id]['nodenumber'],
}))
// contextmenu event
@@ -494,6 +498,9 @@ JSP.prototype.removeNodes = function ($id) {
// delete dom
$(`#${$id}`).remove()
+
+ // callback onRemoveNodes event
+ this.options&&this.options.onRemoveNodes&&this.options.onRemoveNodes($id)
}
/**
@@ -511,6 +518,9 @@ JSP.prototype.removeConnect = function ($connect) {
targetarr = _.filter(targetarr, v => v !== sourceId)
$(`#${targetId}`).attr('data-targetarr', targetarr.toString())
}
+ if ($(`#${sourceId}`).attr('data-tasks-type')=='CONDITIONS') {
+ $(`#${sourceId}`).attr('data-nodenumber',Number($(`#${sourceId}`).attr('data-nodenumber'))-1)
+ }
this.JspInstance.deleteConnection($connect)
this.selectedElement = {}
@@ -566,6 +576,7 @@ JSP.prototype.copyNodes = function ($id) {
[newId]: {
name: newName,
targetarr: '',
+ nodenumber: 0,
x: newX,
y: newY
}
@@ -652,6 +663,7 @@ JSP.prototype.saveStore = function () {
locations[v.id] = {
name: v.name,
targetarr: v.targetarr,
+ nodenumber: v.nodenumber,
x: v.x,
y: v.y
}
@@ -705,6 +717,12 @@ JSP.prototype.handleEvent = function () {
return false
}
+ if ($(`#${sourceId}`).attr('data-tasks-type')=='CONDITIONS' && $(`#${sourceId}`).attr('data-nodenumber')==2) {
+ return false
+ } else {
+ $(`#${sourceId}`).attr('data-nodenumber',Number($(`#${sourceId}`).attr('data-nodenumber'))+1)
+ }
+
// Storage node dependency information
saveTargetarr(sourceId, targetId)
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js
old mode 100644
new mode 100755
index c10dfda5d6..4b485fec0b
--- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js
@@ -43,9 +43,9 @@ const rtBantpl = () => {
/**
* return node html
*/
-const rtTasksTpl = ({ id, name, x, y, targetarr, isAttachment, taskType, runFlag }) => {
+const rtTasksTpl = ({ id, name, x, y, targetarr, isAttachment, taskType, runFlag, nodenumber }) => {
let tpl = ``
- tpl += `
`
+ tpl += `
`
tpl += `
`
tpl += `
`
tpl += `
`
@@ -73,6 +73,7 @@ const tasksAll = () => {
id: e.attr('id'),
name: e.find('.name-p').text(),
targetarr: e.attr('data-targetarr') || '',
+ nodenumber: e.attr('data-nodenumber'),
x: parseInt(e.css('left'), 10),
y: parseInt(e.css('top'), 10)
})
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toobar_CONDITIONS.png b/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toobar_CONDITIONS.png
new file mode 100644
index 0000000000..e8c5e38339
Binary files /dev/null and b/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toobar_CONDITIONS.png differ
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_SQOOP.png b/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_SQOOP.png
new file mode 100644
index 0000000000..2ab3b6bd4a
Binary files /dev/null and b/dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_SQOOP.png differ
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/index.vue
index 2f30f0bea4..1ef2e1f3e4 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/index.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/index.vue
@@ -54,6 +54,9 @@
+
+
+
@@ -80,7 +83,9 @@
// search value
searchVal: '',
// host
- host: ''
+ host: '',
+ // executor name
+ executorName: ''
}
}
},
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue
index 1410a67f90..57ae6bd685 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue
@@ -28,6 +28,9 @@
{{$t('Process Name')}}
|
+
+ {{$t('Executor')}}
+ |
{{$t('Run Type')}}
|
@@ -67,6 +70,10 @@
{{item.name}}
|
+
+ {{item.executorName}}
+ -
+ |
{{_rtRunningType(item.commandType)}} |
{{item.scheduleTime | formatDate}}
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue
index 7bcf9ac26b..b95d4ed720 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue
@@ -71,7 +71,9 @@
// Start Time
startDate: '',
// End Time
- endDate: ''
+ endDate: '',
+ // Exectuor Name
+ executorName: ''
}
}
},
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue
index bb972c3f3b..f7be553568 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue
@@ -28,6 +28,9 @@
|
{{$t('Process Instance')}}
|
+
+ {{$t('Executor')}}
+ |
{{$t('Node Type')}}
|
@@ -64,6 +67,10 @@
{{item.name}}
{{item.processInstanceName}} |
+
+ {{item.executorName}}
+ -
+ |
{{item.taskType}} |
|
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue
index a23eee5fa0..4cb166647e 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue
@@ -68,7 +68,9 @@
// start date
startDate: '',
// end date
- endDate: ''
+ endDate: '',
+ // Exectuor Name
+ executorName: ''
}
}
},
@@ -147,4 +149,4 @@
},
components: { mList, mInstanceConditions, mSpin, mListConstruction, mSecondaryMenu, mNoData }
}
-
\ No newline at end of file
+
diff --git a/dolphinscheduler-ui/src/js/conf/home/store/dag/mutations.js b/dolphinscheduler-ui/src/js/conf/home/store/dag/mutations.js
old mode 100644
new mode 100755
index 6ceabed8c1..b914b86740
--- a/dolphinscheduler-ui/src/js/conf/home/store/dag/mutations.js
+++ b/dolphinscheduler-ui/src/js/conf/home/store/dag/mutations.js
@@ -134,6 +134,7 @@ export default {
state.locations[payload.id] = _.assign(state.locations[payload.id], {
name: dom.find('.name-p').text(),
targetarr: dom.attr('data-targetarr'),
+ nodenumber: dom.attr('data-nodenumber'),
x: parseInt(dom.css('left'), 10),
y: parseInt(dom.css('top'), 10)
})
diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
old mode 100644
new mode 100755
index 0402d7e398..e8ac57adc0
--- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
+++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
@@ -240,6 +240,7 @@ export default {
'Service-Master': 'Service-Master',
'Service-Worker': 'Service-Worker',
'Process Name': 'Process Name',
+ 'Executor': 'Executor',
'Run Type': 'Run Type',
'Scheduling Time': 'Scheduling Time',
'Run Times': 'Run Times',
@@ -518,5 +519,54 @@ export default {
'SpeedRecord': 'speed(record count)',
'0 means unlimited by byte': '0 means unlimited',
'0 means unlimited by count': '0 means unlimited',
- 'Modify User': 'Modify User'
+ 'Modify User': 'Modify User',
+ 'Please enter Mysql Database(required)': 'Please enter Mysql Database(required)',
+ 'Please enter Mysql Table(required)': 'Please enter Mysql Table(required)',
+ 'Please enter Columns (Comma separated)': 'Please enter Columns (Comma separated)',
+ 'Please enter Target Dir(required)': 'Please enter Target Dir(required)',
+ 'Please enter Export Dir(required)': 'Please enter Export Dir(required)',
+ 'Please enter Hive Database(required)': 'Please enter Hive Databasec(required)',
+ 'Please enter Hive Table(required)': 'Please enter Hive Table(required)',
+ 'Please enter Hive Partition Keys': 'Please enter Hive Partition Key',
+ 'Please enter Hive Partition Values': 'Please enter Partition Value',
+ 'Please enter Replace Delimiter': 'Please enter Replace Delimiter',
+ 'Please enter Fields Terminated': 'Please enter Fields Terminated',
+ 'Please enter Lines Terminated': 'Please enter Lines Terminated',
+ 'Please enter Concurrency': 'Please enter Concurrency',
+ 'Please enter Update Key': 'Please enter Update Key',
+ 'Direct': 'Direct',
+ 'Type': 'Type',
+ 'ModelType': 'ModelType',
+ 'ColumnType': 'ColumnType',
+ 'Database': 'Database',
+ 'Column': 'Column',
+ 'Map Column Hive': 'Map Column Hive',
+ 'Map Column Java': 'Map Column Java',
+ 'Export Dir': 'Export Dir',
+ 'Hive partition Keys': 'Hive partition Keys',
+ 'Hive partition Values': 'Hive partition Values',
+ 'FieldsTerminated': 'FieldsTerminated',
+ 'LinesTerminated': 'LinesTerminated',
+ 'IsUpdate': 'IsUpdate',
+ 'UpdateKey': 'UpdateKey',
+ 'UpdateMode': 'UpdateMode',
+ 'Target Dir': 'Target Dir',
+ 'DeleteTargetDir': 'DeleteTargetDir',
+ 'FileType': 'FileType',
+ 'CompressionCodec': 'CompressionCodec',
+ 'CreateHiveTable': 'CreateHiveTable',
+ 'DropDelimiter': 'DropDelimiter',
+ 'OverWriteSrc': 'OverWriteSrc',
+ 'ReplaceDelimiter': 'ReplaceDelimiter',
+ 'Concurrency': 'Concurrency',
+ 'Form': 'Form',
+ 'OnlyUpdate': 'OnlyUpdate',
+ 'AllowInsert': 'AllowInsert',
+ 'Data Source': 'Data Source',
+ 'Data Target': 'Data Target',
+ 'All Columns': 'All Columns',
+ 'Some Columns': 'Some Columns',
+ 'Modify User': 'Modify User',
+ 'Branch flow': 'Branch flow',
+ 'Cannot select the same node for successful branch flow and failed branch flow': 'Cannot select the same node for successful branch flow and failed branch flow'
}
diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
old mode 100644
new mode 100755
index 95eb4a1081..c72090657b
--- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
+++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
@@ -239,6 +239,7 @@ export default {
'Service-Master': '服务管理-Master',
'Service-Worker': '服务管理-Worker',
'Process Name': '工作流名称',
+ 'Executor': '执行用户',
'Run Type': '运行类型',
'Scheduling Time': '调度时间',
'Run Times': '运行次数',
@@ -518,5 +519,54 @@ export default {
'SpeedRecord': '限流(记录数)',
'0 means unlimited by byte': 'KB，0代表不限制',
'0 means unlimited by count': '0代表不限制',
- 'Modify User': '修改用户'
+ 'Modify User': '修改用户',
+ 'Please enter Mysql Database(required)': '请输入Mysql数据库(必填)',
+ 'Please enter Mysql Table(required)': '请输入Mysql表名(必填)',
+ 'Please enter Columns (Comma separated)': '请输入列名，用 , 隔开',
+ 'Please enter Target Dir(required)': '请输入目标路径(必填)',
+ 'Please enter Export Dir(required)': '请输入数据源路径(必填)',
+ 'Please enter Hive Database(required)': '请输入Hive数据库(必填)',
+ 'Please enter Hive Table(required)': '请输入Hive表名(必填)',
+ 'Please enter Hive Partition Keys': '请输入分区键',
+ 'Please enter Hive Partition Values': '请输入分区值',
+ 'Please enter Replace Delimiter': '请输入替换分隔符',
+ 'Please enter Fields Terminated': '请输入列分隔符',
+ 'Please enter Lines Terminated': '请输入行分隔符',
+ 'Please enter Concurrency': '请输入并发度',
+ 'Please enter Update Key': '请输入更新列',
+ 'Direct': '流向',
+ 'Type': '类型',
+ 'ModelType': '模式',
+ 'ColumnType': '列类型',
+ 'Database': '数据库',
+ 'Column': '列',
+ 'Map Column Hive': 'Hive类型映射',
+ 'Map Column Java': 'Java类型映射',
+ 'Export Dir': '数据源路径',
+ 'Hive partition Keys': 'Hive 分区键',
+ 'Hive partition Values': 'Hive 分区值',
+ 'FieldsTerminated': '列分隔符',
+ 'LinesTerminated': '行分隔符',
+ 'IsUpdate': '是否更新',
+ 'UpdateKey': '更新列',
+ 'UpdateMode': '更新类型',
+ 'Target Dir': '目标路径',
+ 'DeleteTargetDir': '是否删除目录',
+ 'FileType': '保存格式',
+ 'CompressionCodec': '压缩类型',
+ 'CreateHiveTable': '是否创建新表',
+ 'DropDelimiter': '是否删除分隔符',
+ 'OverWriteSrc': '是否覆盖数据源',
+ 'ReplaceDelimiter': '替换分隔符',
+ 'Concurrency': '并发度',
+ 'Form': '表单',
+ 'OnlyUpdate': '只更新',
+ 'AllowInsert': '无更新便插入',
+ 'Data Source': '数据来源',
+ 'Data Target': '数据目的',
+ 'All Columns': '全表导入',
+ 'Some Columns': '选择列',
+ 'Modify User': '修改用户',
+ 'Branch flow': '分支流转',
+ 'Cannot select the same node for successful branch flow and failed branch flow': '成功分支流转和失败分支流转不能选择同一个节点'
}
diff --git a/e2e/pom.xml b/e2e/pom.xml
new file mode 100644
index 0000000000..ea67c0ca16
--- /dev/null
+++ b/e2e/pom.xml
@@ -0,0 +1,137 @@
+
+
+ 4.0.0
+ org.apache.dolphinscheduler-e2e
+ dolphinscheduler-e2e
+ 1.0.0
+
+
+ 6.14.3
+ 3.141.59
+ 22.0
+ 2.6
+ 1.1.4
+ 2.8.0
+ 4.2.1
+ testng.xml
+
+
+
+
+
+ org.seleniumhq.selenium
+ selenium-java
+ ${selenium.version}
+
+
+ com.google.guava
+ guava
+ ${guava.version}
+
+
+ com.google.inject
+ guice
+ ${inject.version}
+
+
+
+ org.testng
+ testng
+ ${testng.version}
+
+
+
+ org.uncommons
+ reportng
+ ${reportng.version}
+ test
+
+
+ org.testng
+ testng
+
+
+
+
+
+ commons-io
+ commons-io
+ ${commons-io.version}
+
+
+
+ org.apache.servicemix.bundles
+ org.apache.servicemix.bundles.jedis
+ 2.6.2_1
+
+
+ org.apache.commons
+ commons-pool2
+ ${commons-pool2.version}
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.8.0
+
+ 1.8
+ 1.8
+ UTF-8
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+ 2.20.1
+
+
+ ${xml.file}
+
+ -Dfile.encoding=UTF-8
+
+
+ usedefaultlisteners
+
+ false
+
+
+ listener
+ org.uncommons.reportng.HTMLReporter,org.uncommons.reportng.JUnitXMLReporter
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/e2e/src/main/java/org/apache/dolphinscheduler/constant/TestConstant.java b/e2e/src/main/java/org/apache/dolphinscheduler/constant/TestConstant.java
new file mode 100644
index 0000000000..8a3b8eb6b3
--- /dev/null
+++ b/e2e/src/main/java/org/apache/dolphinscheduler/constant/TestConstant.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.constant;
+
+public class TestConstant {
+ /**
+ * 1000
+ */
+ public static final int ONE_THOUSANG = 1000;
+
+
+
+ /**
+ * 3000
+ */
+ public static final int THREE_THOUSANG = 3000;
+
+ /**
+ * 10000
+ */
+ public static final int TEN_THOUSANG = 10000;
+
+}
diff --git a/e2e/src/main/java/org/apache/dolphinscheduler/util/PropertiesReader.java b/e2e/src/main/java/org/apache/dolphinscheduler/util/PropertiesReader.java
new file mode 100644
index 0000000000..cbf7d07bd7
--- /dev/null
+++ b/e2e/src/main/java/org/apache/dolphinscheduler/util/PropertiesReader.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.util;
+
+import java.io.*;
+import java.util.Properties;
+
+/**
+ * read properties
+ */
+public class PropertiesReader {
+ private static Properties properties = new Properties();
+
+ /**
+ * @param propertiesPath properties path
+ * @return Properties
+ * @throws IOException IOException
+ */
+ public static Properties readProperties(String propertiesPath) throws IOException {
+ System.out.println("read properties ");
+ InputStream inputStream = new FileInputStream(propertiesPath);
+ InputStreamReader inputStreamReader = new InputStreamReader(inputStream, "UTF-8");
+ BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
+ properties.load(bufferedReader);
+ return properties;
+ }
+
+ /***
+ * @param key key
+ * @return value
+ */
+ public static String getKey(String key) {
+ return properties.getProperty(key);
+ }
+}
diff --git a/e2e/src/main/java/org/apache/dolphinscheduler/util/RedisUtil.java b/e2e/src/main/java/org/apache/dolphinscheduler/util/RedisUtil.java
new file mode 100644
index 0000000000..7f9340f0ca
--- /dev/null
+++ b/e2e/src/main/java/org/apache/dolphinscheduler/util/RedisUtil.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.util;
+
+import org.apache.dolphinscheduler.constant.TestConstant;
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
+import redis.clients.jedis.JedisPoolConfig;
+
+/**
+ * redis util
+ */
+public class RedisUtil {
+ /**
+ * redis ip
+ */
+ private static String redisIp;
+
+ /**
+ * redis port
+ */
+ private static int redisPort;
+
+ /**
+ * redis password
+ */
+ private static String redisPwd;
+
+ /**
+ * redis pool config
+ */
+ private static JedisPoolConfig jedisPoolConfig;
+
+ /**
+ * redis pool
+ */
+ private static JedisPool jedisPool;
+
+ /**
+ * jedis connection
+ */
+ private Jedis jedis;
+
+ /**
+ * jedis expire time
+ */
+ private int jedisExpireTime;
+
+ /**
+ * jedis max total
+ */
+ private static int jedisPoolMaxTotal;
+
+ /**
+ * jedis max idle
+ */
+ private static int jedisPoolMaxIdle;
+
+ /**
+ * jedis max wait time
+ */
+ private static int jedisPoolMaxWaitMillis;
+
+ /**
+ * Whether to perform a valid check when calling the borrowObject method
+ */
+ private static boolean jedisPoolTestOnBorrow;
+
+ /**
+ * Whether to perform a valid check when calling the returnObject method
+ */
+ private static boolean jedisPoolTestOnReturn;
+
+ /**
+ * storage local thread
+ */
+ public static ThreadLocal threadLocal = new ThreadLocal<>();
+
+ /*
+ * redis init
+ */
+ static {
+ // redis properties
+ redisIp = PropertiesReader.getKey("redis.ip");
+ redisPort = Integer.valueOf(PropertiesReader.getKey("redis.port"));
+ redisPwd = PropertiesReader.getKey("redis.pwd");
+ //redis pool properties
+ jedisPoolMaxTotal = Integer.valueOf(PropertiesReader.getKey("jedis.pool.maxTotal"));
+ jedisPoolMaxIdle = Integer.valueOf(PropertiesReader.getKey("jedis.pool.maxIdle"));
+ jedisPoolMaxWaitMillis = Integer.valueOf(PropertiesReader.getKey("jedis.pool.maxWaitMillis"));
+ jedisPoolTestOnBorrow = Boolean.valueOf(PropertiesReader.getKey("jedis.pool.testOnBorrow"));
+ jedisPoolTestOnReturn = Boolean.valueOf(PropertiesReader.getKey("jedis.pool.testOnReturn"));
+ // redis pool start properties
+ jedisPoolConfig = new JedisPoolConfig();
+ jedisPoolConfig.setMaxTotal(jedisPoolMaxTotal);
+ jedisPoolConfig.setMaxIdle(jedisPoolMaxIdle);
+ jedisPoolConfig.setMaxWaitMillis(jedisPoolMaxWaitMillis);
+ jedisPoolConfig.setTestOnBorrow(jedisPoolTestOnBorrow);
+ jedisPoolConfig.setTestOnReturn(jedisPoolTestOnReturn);
+ // connect redis
+ try {
+ System.out.println("redis init");
+ if (redisPwd.isEmpty())
+ jedisPool = new JedisPool(jedisPoolConfig, redisIp, redisPort, TestConstant.THREE_THOUSANG);
+ else {
+ jedisPool = new JedisPool(jedisPoolConfig, redisIp, redisPort, TestConstant.TEN_THOUSANG, redisPwd);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ System.out.println("redis connect fail");
+ }
+ }
+
+ /**
+ * get redis pool
+ *
+ * @return redis pool
+ */
+ public static JedisPool getJedisPool() {
+ return jedisPool;
+ }
+
+ /**
+ * get jedis connection
+ *
+ * @return jedis connection
+ */
+ public Jedis getNewJedis() {
+ Jedis newJedis = null;
+ try {
+ newJedis = jedisPool.getResource();
+ } catch (Exception e) {
+ e.printStackTrace();
+ System.out.println("redis connection fail");
+ }
+ System.out.println("redis connection success");
+ return newJedis;
+ }
+
+ /**
+ * get jedis connection
+ *
+ * @return jedis connection
+ */
+ public Jedis getJedis() {
+ return jedis;
+ }
+
+ public void setJedisAndExpire(Jedis jedis) {
+ this.jedis = jedis;
+ threadLocal.set(jedis);
+ // jedis expire time(s)
+ jedisExpireTime = Integer.valueOf(PropertiesReader.getKey("jedis.expireTime"));
+ System.out.println("redisUtil sets up a redis connection");
+ }
+
+ /**
+ * set key
+ *
+ * @param key key
+ * @param value value
+ *
+ */
+
+ public void setKey(String key, String value) {
+ jedis.set(key, value);
+ // set expire time 1h
+ jedis.expire(key, jedisExpireTime);
+ }
+
+ /**
+ * get key
+ *
+ * @param key key
+ * @return value
+ */
+ public String getKey(String key) {
+ return jedis.get(key);
+ }
+
+ /**
+ * Return jedis connection
+ */
+ public void returnJedis() {
+ if (jedis != null) {
+ jedis.close();
+ }
+ System.out.println("jedis has been returned");
+ }
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java
new file mode 100644
index 0000000000..7d3ab9b837
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.base;
+
+
+import org.apache.dolphinscheduler.constant.TestConstant;
+import org.apache.dolphinscheduler.util.PropertiesReader;
+import org.openqa.selenium.JavascriptExecutor;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import org.openqa.selenium.PageLoadStrategy;
+
+
+/**
+ * base driver class
+ */
+public class BaseDriver {
+ /**
+ * driver
+ */
+ private WebDriver driver;
+
+ /**
+ * chrome driver path
+ */
+ private String chromeDriverPath;
+
+
+ /**
+ * implicitly wait time（s）
+ */
+ private long implicitlyWait;
+
+ /**
+ * page load timeout（s）
+ */
+ private long pageLoadTimeout;
+
+ /**
+ * script Timeout（s）
+ */
+ private long setScriptTimeout;
+
+
+ /**
+ * Local thread storage is used to store the driver
+ */
+ public static ThreadLocal threadLocal = new ThreadLocal<>();
+
+ /**
+ *Initialization parameters
+ */
+ public BaseDriver() throws IOException {
+ /* driver test class path */
+ chromeDriverPath = PropertiesReader.getKey("driver.chromeDriver");
+
+ /* wait time */
+ implicitlyWait = Long.valueOf(PropertiesReader.getKey("driver.timeouts.implicitlyWait"));
+ pageLoadTimeout = Long.valueOf(PropertiesReader.getKey("driver.timeouts.pageLoadTimeout"));
+ setScriptTimeout = Long.valueOf(PropertiesReader.getKey("driver.timeouts.setScriptTimeout"));
+ }
+
+
+ /**
+ * start chrome browser
+ */
+ public void startBrowser() throws Exception {
+ // set chrome driver
+ System.setProperty("webdriver.chrome.driver", chromeDriverPath);
+ ChromeOptions chromeOptions = new ChromeOptions();
+ chromeOptions.setPageLoadStrategy(PageLoadStrategy.NONE);
+ chromeOptions.addArguments("--no-sandbox");
+ chromeOptions.addArguments("--disable-dev-shm-usage");
+ chromeOptions.addArguments("--headless");
+ chromeOptions.addArguments("--disable-gpu");
+ chromeOptions.addArguments("--whitelisted-ips");
+ chromeOptions.addArguments("--disable-infobars");
+ chromeOptions.addArguments("--disable-browser-side-navigation");
+ driver = new ChromeDriver(chromeOptions);
+
+ /* driver setting wait time */
+ // implicitly wait time
+ driver.manage().timeouts().implicitlyWait(implicitlyWait, TimeUnit.SECONDS);
+
+ // page load timeout
+ driver.manage().timeouts().pageLoadTimeout(pageLoadTimeout, TimeUnit.SECONDS);
+
+ // page load timeout
+ driver.manage().timeouts().pageLoadTimeout(pageLoadTimeout, TimeUnit.SECONDS);
+
+ // script timeout
+ driver.manage().timeouts().setScriptTimeout(setScriptTimeout, TimeUnit.SECONDS);
+
+ // window maximize
+ driver.manage().window().maximize();
+
+ // set threadLocal
+ threadLocal.set(driver);
+ }
+
+ /**
+ * get webDriver
+ *
+ * @return driver
+ */
+ public WebDriver getDriver() {
+ return driver;
+ }
+
+ /**
+ * set webDriver
+ *
+ * @param driver driver
+ */
+ public void setDriver(WebDriver driver) {
+ this.driver = driver;
+ // Thread local storage
+ threadLocal.set(driver);
+ }
+
+ /**
+ * close browser
+ */
+ public void closeBrowser() throws InterruptedException {
+ // JS Show a pop-up box to indicate the end of the test
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+ ((JavascriptExecutor) driver).executeScript("alert('Test completed, browser closes after 3s')");
+ Thread.sleep(TestConstant.THREE_THOUSANG);
+ if (driver != null) {
+ driver.quit();
+ }
+ }
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseTest.java
new file mode 100644
index 0000000000..c12c19fc1d
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.base;
+
+
+import org.apache.dolphinscheduler.page.LoginPage;
+import org.apache.dolphinscheduler.util.PropertiesReader;
+import org.openqa.selenium.WebDriver;
+import org.testng.annotations.*;
+
+import java.io.IOException;
+import java.util.Properties;
+
+/**
+ * base test class
+ */
+public class BaseTest {
+ /**
+ * properties
+ */
+ private static Properties properties;
+
+
+ /**
+ * baseDriver
+ */
+ private BaseDriver baseDriver;
+
+ /**
+ * driver
+ */
+ public WebDriver driver;
+
+ /**
+ * Executed before executing a test suite
+ * Read the test configuration file
+ *
+ * @param propertiesPath properties path
+ * @throws IOException IOException
+ */
+ @BeforeSuite(alwaysRun = true)
+ @Parameters({"propertiesPath"})
+ public void beforeSuite(@Optional("src/test/resources/config/config.properties") String propertiesPath) throws IOException {
+ // read properties
+ properties = PropertiesReader.readProperties(propertiesPath);
+ }
+
+ /**
+ * Executed before executing a testcase
+ */
+ @BeforeTest(alwaysRun = true)
+ public void beforeTest() throws Exception {
+ //base driver
+ baseDriver = new BaseDriver();
+ baseDriver.startBrowser();
+ driver = baseDriver.getDriver();
+ }
+
+ /**
+ * Executed before executing a class method in a test case
+ */
+ @BeforeClass(alwaysRun = true)
+ public void setUp() throws IOException, InterruptedException {
+ LoginPage loginPage = new LoginPage(driver);
+ loginPage.jumpPage();
+ loginPage.login();
+ }
+
+
+ /**
+ * Execute after executing a class method in a test case
+ */
+ @AfterClass(alwaysRun = true)
+ public void afterClass() {
+ // logout
+ }
+
+ /**
+ * Execute after executing a testcase
+ */
+ @AfterTest(alwaysRun = true)
+ public void afterTest() throws InterruptedException {
+ // close browser
+ baseDriver.closeBrowser();
+ }
+
+ /**
+ * Execute after executing a testsuite
+ */
+ @AfterSuite(alwaysRun = true)
+ public void afterSuite() {
+ }
+}
\ No newline at end of file
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java b/e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java
new file mode 100644
index 0000000000..072ccb652e
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.common;
+
+import org.apache.dolphinscheduler.util.PropertiesReader;
+import org.apache.dolphinscheduler.util.RedisUtil;
+import org.openqa.selenium.*;
+import org.openqa.selenium.interactions.Actions;
+import org.openqa.selenium.support.ui.ExpectedConditions;
+import org.openqa.selenium.support.ui.WebDriverWait;
+import redis.clients.jedis.Jedis;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Encapsulate the most basic operations on the interface in the browser
+ */
+public class BrowserCommon {
+ /**
+ * driver
+ */
+ protected WebDriver driver;
+
+ /**
+ * actions
+ */
+ protected Actions actions;
+
+ /**
+ * Javascript
+ */
+ protected JavascriptExecutor je;
+
+ /**
+ * Show wait
+ */
+ protected WebDriverWait wait;
+
+ /**
+ * Jedis
+ */
+ protected Jedis jedis;
+
+ /**
+ * redis util
+ */
+ protected RedisUtil redisUtil;
+
+ /**
+ * @param driver driver
+ */
+ public BrowserCommon(WebDriver driver) {
+ this.driver = driver;
+ this.actions = new Actions(driver);
+ this.je = ((JavascriptExecutor) driver);
+ // show wait timeout
+ long timeout = Long.valueOf(PropertiesReader.getKey("driver.timeouts.webDriverWait"));
+ wait = new WebDriverWait(driver, timeout);
+ }
+
+ /**
+ * @param driver driver
+ * @param jedis jedis
+ */
+ public BrowserCommon(WebDriver driver, Jedis jedis) {
+ this.driver = driver;
+ this.actions = new Actions(driver);
+ this.je = ((JavascriptExecutor) driver);
+ // show wait timeout
+ long timeout = Long.valueOf(PropertiesReader.getKey("driver.timeouts.webDriverWait"));
+ wait = new WebDriverWait(driver, timeout);
+ this.jedis = jedis;
+ }
+
+ /**
+ * @param driver driver
+ * @param redisUtil redisUtil
+ */
+ public BrowserCommon(WebDriver driver, RedisUtil redisUtil) {
+ this.driver = driver;
+ this.actions = new Actions(driver);
+ this.je = ((JavascriptExecutor) driver);
+ // show wait timeout
+ long timeout = Long.valueOf(PropertiesReader.getKey("driver.timeouts.webDriverWait"));
+ wait = new WebDriverWait(driver, timeout);
+ }
+
+
+ /**
+ * Get WebElement element object through element positioning
+ *
+ * @param locator By
+ * @return WebElement
+ */
+
+ public WebElement locateElement(By locator) {
+ return wait.until(ExpectedConditions.presenceOfElementLocated(locator));
+ }
+
+ /**
+ * Click button element
+ * @param locator By
+ * @return clickButton
+ */
+ public WebElement clickButton(By locator) {
+ WebElement buttonElement = locateElement(locator);
+ wait.until(ExpectedConditions.elementToBeClickable(locator));
+ ExpectedConditions.elementToBeClickable(locator);
+ buttonElement.click();
+ return buttonElement;
+ }
+
+ /**
+ * Click element
+ *
+ * @param locator By
+ * @return inputElement
+ */
+ public WebElement clickElement(By locator) {
+ WebElement clickElement = locateElement(locator);
+ clickElement.click();
+ return clickElement;
+ }
+
+ /**
+ * input element
+ *
+ * @param locator By
+ * @param content Input content
+ * @return inputElement
+ */
+ public WebElement sendInput(By locator, String content) {
+ WebElement inputElement = locateElement(locator);
+ inputElement.clear();
+ inputElement.sendKeys(content);
+ return inputElement;
+ }
+ /**
+ * clear element
+ *
+ * @param locator By
+ */
+ public WebElement clearInput(By locator) {
+ WebElement clearElement = locateElement(locator);
+ clearElement.click();
+ clearElement.sendKeys(Keys.chord(Keys.CONTROL, "a"));
+ clearElement.sendKeys(Keys.BACK_SPACE);
+ return clearElement;
+ }
+
+ /**
+ * input codeMirror
+ *
+ * @param codeMirrorLocator By codeMirror
+ * @param codeMirrorLineLocator By codeMirrorLine
+
+ */
+ public void inputCodeMirror(By codeMirrorLocator,By codeMirrorLineLocator,String content) {
+ WebElement codeMirrorElement = locateElement(codeMirrorLocator);
+ WebElement codeMirrorLineElement = locateElement(codeMirrorLineLocator);
+ codeMirrorElement.click();
+ codeMirrorLineElement.sendKeys(content);
+ }
+
+ /**
+ * move to element
+ * @param locator BY
+ * @return actions
+ */
+ public Actions moveToElement(By locator){
+ return actions.moveToElement(locateElement(locator));
+ }
+
+ /**
+ * mouse drag element
+ *
+ * @param source_locator BY
+ * @param target_locator BY
+ */
+ public void dragAndDrop(By source_locator, By target_locator){
+ WebElement sourceElement = locateElement(source_locator);
+ WebElement targetElement = locateElement(target_locator);
+ actions.dragAndDrop(sourceElement, targetElement).perform();
+ actions.release();
+ }
+
+ public void moveToDragElement(By target_locator, int X, int Y){
+ WebElement targetElement = locateElement(target_locator);
+ actions.dragAndDropBy(targetElement, X, Y).perform();
+ actions.release();
+ }
+
+
+ /**
+ * jump page
+ *
+ * @param url url
+ */
+ public void jumpPage(String url) {
+ driver.get(url);
+ }
+
+
+ /**
+ * Find the next handle, recommended for two windows
+ *
+ * @return driver
+ */
+ public WebDriver switchNextHandle() {
+ // Current window handle
+ String currentHandle = driver.getWindowHandle();
+ // All window handle
+ Set allHandles = driver.getWindowHandles();
+ // Finding the next handle
+ for (String handle : allHandles) {
+ if (!handle.equals(currentHandle)) {
+ return driver.switchTo().window(handle);
+ }
+ }
+ return driver;
+ }
+
+ /**
+ * Multi-window switch handle, according to the handle number passed in
+ *
+ * @param num Number starts from 1
+ * @return driver
+ */
+ public WebDriver switchHandle(int num) {
+ // current handle
+ String currentHandle = driver.getWindowHandle();
+ // all handle
+ Set allHandlesSet = driver.getWindowHandles();
+ List allHandlesList = new ArrayList<>(allHandlesSet);
+ // switch handle
+ return driver.switchTo().window(allHandlesList.get(num - 1));
+ }
+
+ /**
+ * Switch frame structure
+ *
+ * @param locator frame
+ * @return driver
+ */
+ public WebDriver switchFrame(By locator) {
+ return driver.switchTo().frame(locateElement(locator));
+ }
+
+ /**
+ * Switch parent frame structure
+ *
+ * @return driver
+ */
+ public WebDriver switchParentFrame() {
+ return driver.switchTo().parentFrame();
+ }
+
+ /**
+ * Switch out of frame structure
+ *
+ * @return driver
+ */
+ public WebDriver switchOutOfFrame() {
+ return driver.switchTo().defaultContent();
+ }
+
+
+ /**
+ * execute JS Script
+ *
+ * @param script JS script
+ */
+ public void executeScript(String script) {
+ je.executeScript(script);
+ }
+
+ /**
+ * execute JS Script
+ *
+ * @param script JS script
+ * @param args Object element array
+ */
+ public void executeScript(String script, Object... args) {
+ je.executeScript(script, args);
+ }
+
+ /**
+ * Page slide to top
+ */
+ public void scrollToTop() {
+ executeScript("window.scrollTo(0, 0)");
+ }
+
+ /**
+ * Page slides to the bottom
+ */
+ public void scrollToBottom() {
+ executeScript("window.scrollTo(0, document.body.scrollHeight)");
+ }
+
+ public void scrollToElementBottom() {
+
+ WebElement webElement = driver.findElement(By.xpath("/html/body/div[4]/div/div[2]/div/div[2]/div/div[7]/div[3]"));
+ ((JavascriptExecutor) driver).executeScript("arguments[0].scrollIntoView(true);", webElement);
+ }
+
+
+ /**
+ * Page swipe makes the top of the element align with the top of the page
+ *
+ * @param by Elements that need to be aligned with the top of the page
+ */
+ public void scrollElementTopToTop(By by) {
+ executeScript("arguments[0].scrollIntoView(true);", driver.findElement(by));
+ }
+
+ /**
+ * Page sliding makes the bottom of the element aligned with the bottom of the page
+ *
+ * @param by Elements that need to be aligned with the bottom of the page
+ */
+ public void scrollElementBottomToBottom(By by) {
+ executeScript("arguments[0].scrollIntoView(false);", driver.findElement(by));
+ }
+
+
+ /**
+ * Determine if the current page title is the specified title
+ *
+ * @param title title
+ * @return boolean
+ */
+
+ public boolean ifTitleIs(String title) {
+ return wait.until(ExpectedConditions.titleIs(title));
+ }
+
+ /**
+ * Determines whether the current page title contains the specified text
+ *
+ * @param text text
+ * @return boolean
+ */
+ public boolean ifTitleContains(String text) {
+ return wait.until(ExpectedConditions.titleContains(text));
+ }
+
+ /**
+ * Determines whether the text value of an element on the current page is the specified text
+ *
+ * @param locator By
+ * @param text text
+ * @return boolean
+ */
+ public boolean ifTextExists(By locator, String text) {
+ return wait.until(ExpectedConditions.textToBePresentInElementLocated(locator, text));
+ }
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/common/PageCommon.java b/e2e/src/test/java/org/apache/dolphinscheduler/common/PageCommon.java
new file mode 100644
index 0000000000..72dc03a77c
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/common/PageCommon.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.common;
+
+import org.apache.dolphinscheduler.util.RedisUtil;
+import org.openqa.selenium.WebDriver;
+import redis.clients.jedis.Jedis;
+
+
+/**
+ * Encapsulate the operation methods that can be used for each module page
+ */
+public class PageCommon extends BrowserCommon {
+ /**
+ * @param driver driver
+ */
+ public PageCommon(WebDriver driver) {
+ super(driver);
+ }
+
+ /**
+ * @param driver driver
+ * @param jedis jedis
+ */
+ public PageCommon(WebDriver driver, Jedis jedis) {
+ super(driver, jedis);
+ }
+
+ /**
+ * @param driver driver
+ * @param redisUtil redisUtil
+ */
+ public PageCommon(WebDriver driver, RedisUtil redisUtil) {
+ super(driver, redisUtil);
+ }
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java
new file mode 100644
index 0000000000..532849565c
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.data;
+
+import org.apache.dolphinscheduler.util.PropertiesReader;
+
+/**
+ * Landing page object: data
+ */
+
+public class LoginData {
+ /**
+ * Login URL
+ */
+ public static final String URL = PropertiesReader.getKey("LOGIN_URL");
+
+
+ /**
+ * Login username
+ */
+ public static final String USER = PropertiesReader.getKey("USER_NAME");
+
+ /**
+ * Login password
+ */
+ public static final String PASSWORD = PropertiesReader.getKey("PASSWORD");
+
+ public static final String TENANT = "Tenant Manage - DolphinScheduler";
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatProjectData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatProjectData.java
new file mode 100644
index 0000000000..8f6c9c8e97
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatProjectData.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.data.project;
+
+public class CreatProjectData {
+ // create project name
+ public static final String PROJECT_NAME = "selenium_project";
+ // create project description
+ public static final String DESCRIPTION = "test create project description";
+ // project page title
+ public static final String PROJECT_TITLE = "项目 - DolphinScheduler";
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatWorkflowData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatWorkflowData.java
new file mode 100644
index 0000000000..765a54f406
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatWorkflowData.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.data.project;
+
+public class CreatWorkflowData {
+ //input shell task name
+ public static final String SHELL_TASK_NAME = "shell task description test";
+
+ //input shell task description
+ public static final String SHELL_TASK_DESCRIPTION = "shell task description test";
+
+ //input timeout
+ public static final String INPUT_TIMEOUT = "60";
+
+ //input shell script
+ public static final String SHELL_SCRIPT = "echo 1111111";
+
+ //input custom parameters
+ public static final String INPUT_CUSTOM_PARAMETERS = "selenium_parameter";
+
+ //input custom parameters value
+ public static final String INPUT_CUSTOM_PARAMETERS_VALUE = "selenium_parameter_123";
+
+ //input add custom parameters
+ public static final String INPUT_ADD_CUSTOM_PARAMETERS = "selenium_parameter_delete";
+
+ //input add custom parameters value
+ public static final String INPUT_ADD_CUSTOM_PARAMETERS_VALUE = "selenium_parameter_delete_456";
+
+ //create workflow title
+ public static final String WORKFLOW_TITLE = "创建流程定义 - DolphinScheduler";
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java
new file mode 100644
index 0000000000..e6f6ee6b86
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Tenant page object: data
+ */
+package org.apache.dolphinscheduler.data.security;
+
+import org.apache.dolphinscheduler.data.LoginData;
+
+public class TenantManageData {
+ /**
+ * Tenant URL
+ */
+ public static final String TENANAT_URL = LoginData.URL + "/ui/#/security/tenant";
+
+ /**
+ * Tenant Code
+ */
+ public static final String TENANAT_CODE = "dolphinscheduler_tenant_code2";
+
+ /**
+ * Tenant Name
+ */
+ public static final String TENANAT_NAME = "dolphinscheduler_tenant_Name";
+
+ /**
+ * Queue
+ */
+ public static final String QUEUE = "default";
+
+ /**
+ * Description
+ */
+ public static final String DESCRIPTION = "creat tenant test";
+
+ public static final String TENANAT_MANAGE = "Tenant Manage - DolphinScheduler";
+
+
+
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java
new file mode 100644
index 0000000000..03c985fd81
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.data.security;
+
+import org.apache.dolphinscheduler.data.LoginData;
+
+public class UserManageData {
+ public static final String USER_URL = LoginData.URL + "/ui/#/security/users";
+
+ public static final String USERNAME = "selenium111";
+
+ public static final String PASSWORD = "123456qwe";
+
+ public static final String EMAIL = "123456789@qq.com";
+
+ public static final String PHONE = "15811112222";
+
+ public static final String USER_MANAGE = "用户管理 - DolphinScheduler";
+
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/LoginLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/LoginLocator.java
new file mode 100644
index 0000000000..32a82bbbc8
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/LoginLocator.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.locator;
+
+import org.openqa.selenium.By;
+
+/**
+ * Page object: element positioning
+ */
+
+public class LoginLocator {
+ public static final By LOGIN_INPUT_USER = By.xpath("//input[@class='input-element suffix']");
+
+ public static final By LOGIN_INPUT_PASSWORD = By.xpath("//input[@class='input-element suffix']");
+
+ public static final By LOGIN_BUTTON = By.xpath("//button");
+
+ public static final By LOGIN_BUTTON_MOVE = By.xpath("//button[contains(.,' Loading...')]");
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateProjectLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateProjectLocator.java
new file mode 100644
index 0000000000..d2c0d8412c
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateProjectLocator.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.locator.project;
+
+import org.openqa.selenium.By;
+
+public class CreateProjectLocator {
+ //click project manage
+ public static final By PROJECT_MANAGE = By.xpath("//div[2]/div/a/span");
+
+ //click create project button
+ public static final By CREATE_PROJECT_BUTTON = By.xpath("//button/span");
+
+ //input project name
+ public static final By PROJECT_NAME = By.xpath("//div[2]/div/div/div[2]/div/input");
+
+ //input project description
+ public static final By PROJECT_DESCRIPTION = By.xpath("//textarea");
+
+ //submit button
+ public static final By SUBMIT_BUTTON = By.xpath("//div[3]/button[2]/span");
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java
new file mode 100644
index 0000000000..f063d6ef61
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.locator.project;
+
+import org.openqa.selenium.By;
+
+public class CreateWorkflowLocator {
+ // click project name
+ public static final By CLICK_PROJECT_NAME = By.xpath("//span/a");
+
+ // click workflow define
+ public static final By CLICK_WORKFLOW_DEFINE = By.xpath("//li/span");
+
+ // click create workflow button
+ public static final By CLICK_CREATE_WORKFLOW_BUTTON = By.xpath("//button/span");
+
+ //mouse down at shell
+ public static final By MOUSE_DOWN_AT_SHELL = By.xpath("//div[@id='SHELL']/div/div");
+
+ //mouse down at spark
+ public static final By MOUSE_DOWN_AT_SPARK = By.xpath("//div[5]/div/div");
+
+ //mouse move at DAG
+ public static final By MOUSE_MOVE_SHELL_AT_DAG = By.xpath("//div[@id='canvas']");
+
+ //input shell task _name
+ public static final By INPUT_SHELL_TASK_NAME = By.xpath("//input");
+
+ //click stop run type
+ public static final By CLICK_STOP_RUN_TYPE = By.xpath("//label[2]/span/input");
+
+ //click normal run type
+ public static final By CLICK_NORMAL_RUN_TYPE = By.xpath("//span/input");
+
+ //input shell task description
+ public static final By INPUT_SHELL_TASK_DESCRIPTION = By.xpath("//label/div/textarea");
+
+ //click task priority
+ public static final By CLICK_TASK_PRIORITY = By.xpath("//span/div/div/div/div/div");
+
+ //select task priority
+ public static final By SELECT_TASK_PRIORITY = By.xpath("//li[2]/li/span");
+
+ //click work group
+ public static final By CLICK_WORK_GROUP = By.xpath("//div/div/input");
+
+ //select work group
+ public static final By SELECT_WORK_GROUP = By.xpath("//div[4]/div[2]/div/div[1]/div/input");
+
+ //select number of failed retries
+ public static final By SELECT_FAIL_RETRIES_NUMBER = By.xpath("//div[5]/div[2]/div[1]/div[1]/div/input");
+
+ //select failed retry interval
+ public static final By SELECT_FAIL_RETRIES_INTERVAL = By.xpath("//div[5]/div[2]/div[2]/div[1]/div/input");
+
+ //click timeout alarm
+ public static final By CLICK_TIMEOUT_ALARM = By.xpath("//label/div/span/span");
+
+ //select timeout fail
+ public static final By SELECT_TIMEOUT_FAIL = By.xpath("//div/div/label[2]/span/input");
+
+ //cancel timeout alarm
+ public static final By CANCEL_TIMEOUT_ALARM = By.xpath("//div/div/label/span/input");
+
+ //select timeout alarm
+ public static final By SELECT_TIMEOUT_ALARM = By.xpath("//div/div/label/span/input");
+
+ //input timeout
+ public static final By SELECT_TIMEOUT = By.xpath("//div[3]/div[2]/label/div/input");
+
+ //click codeMirror
+ public static final By CLICK_CODE_MIRROR = By.xpath("//div[5]/div/pre");
+
+ //input script
+ public static final By INPUT_SCRIPT = By.xpath("//div[2]/div/div/div/div/div/textarea");
+
+ //click custom parameters
+ public static final By CLICK_CUSTOM_PARAMETERS = By.xpath("//span/a/em");
+
+ //input custom parameters
+ public static final By INPUT_CUSTOM_PARAMETERS = By.xpath("//div[2]/div/div/div/div/div/input");
+
+ //input custom parameters value
+ public static final By INPUT_CUSTOM_PARAMETERS_VALUE = By.xpath("//div[2]/input");
+
+ //click add custom parameters
+ public static final By CLICK_ADD_CUSTOM_PARAMETERS = By.xpath("//span[2]/a/em");
+
+ //input add custom parameters
+ public static final By INPUT_ADD_CUSTOM_PARAMETERS = By.xpath("//div[2]/div/div/div/div[2]/div/input");
+
+ //input add custom parameters value
+ public static final By INPUT_ADD_CUSTOM_PARAMETERS_VALUE = By.xpath("//div[2]/div[2]/input");
+
+ //delete custom parameters
+ public static final By CLICK_DELETE_CUSTOM_PARAMETERS = By.xpath("//div[2]/span/a/em");
+
+ //click submit button
+ public static final By CLICK_SUBMIT_BUTTON = By.xpath("//button[2]/span");
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TenantManageLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TenantManageLocator.java
new file mode 100644
index 0000000000..7d9c8a57ce
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TenantManageLocator.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.locator.security;
+
+import org.openqa.selenium.By;
+
+public class TenantManageLocator{
+ public static final By CREATE_TENANT_BUTTON = By.xpath("//button[@class='ans-btn ans-btn-ghost ans-btn-small']");
+
+ public static final By TENANT_INPUT_CODE = By.xpath("//div[2]/div/div/div[2]/div/input");
+
+ public static final By TENANT_INPUT_NAME = By.xpath("//div[2]/div[2]/div/input");
+
+ public static final By QUEUE = By.xpath("//textarea");
+
+ public static final By DESCRIPTION = By.xpath("//textarea");
+
+ public static final By SUBMIT_BUTTON = By.xpath("//div[3]/button[2]/span");
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java
new file mode 100644
index 0000000000..0d84692cb2
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.locator.security;
+
+import org.openqa.selenium.By;
+
+public class UserManageLocator {
+
+ public static final By CLICK_USER_MANAGE = By.xpath("//div[3]/div/a/div/a/span");
+
+ public static final By CLICK_CREATE_USER_BUTTON = By.xpath("//span[contains(.,'创建用户')]");
+
+ public static final By INPUT_USERNAME = By.xpath("//div[2]/div/div/div[2]/div/input");
+
+ public static final By INPUT_PASSWORD = By.xpath("//div[2]/div[2]/div/input");
+
+ public static final By CLICK_TENANT = By.xpath("//div[3]/div[2]/div/div/div/input");
+
+ public static final By SELECT_TENANT = By.xpath("//div[3]/div[2]/div/div[2]/div/div/div/ul/li/span");
+
+ public static final By CLICK_QUEUE = By.xpath("//div[4]/div[2]/div/div/div/input");
+
+ public static final By SELECT_QUEUE = By.xpath("//div[4]/div[2]/div/div[2]/div/div/div/ul/li/span");
+
+ public static final By TENANT_INPUT_EMAIL = By.xpath("//div[5]/div[2]/div/input");
+
+ public static final By TENANT_INPUT_PHONE = By.xpath("//div[6]/div[2]/div/input");
+
+ public static final By SUBMIT = By.xpath("//div[3]/button[2]/span");
+
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java
new file mode 100644
index 0000000000..cd6b318651
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.page;
+
+import org.apache.dolphinscheduler.common.PageCommon;
+import org.apache.dolphinscheduler.constant.TestConstant;
+import org.apache.dolphinscheduler.data.LoginData;
+import org.apache.dolphinscheduler.locator.LoginLocator;
+import org.apache.dolphinscheduler.util.RedisUtil;
+import org.openqa.selenium.WebDriver;
+
+
+
+public class LoginPage extends PageCommon {
+ /**
+ * Unique constructor
+ * @param driver driver
+ */
+ public LoginPage(WebDriver driver) {
+ super(driver);
+ }
+
+
+ /**
+ * jump page
+ */
+ public void jumpPage() {
+ System.out.println("jump login page");
+ super.jumpPage(LoginData.URL);
+ }
+
+ /**
+ * login
+ *
+ * @return Whether to enter the specified page after searching
+ */
+ public boolean login() throws InterruptedException {
+ System.out.println("LoginPage");
+ // login data
+ sendInput(LoginLocator.LOGIN_INPUT_USER, LoginData.USER);
+ sendInput(LoginLocator.LOGIN_INPUT_PASSWORD, LoginData.PASSWORD);
+
+ // click login button
+ clickButton(LoginLocator.LOGIN_BUTTON);
+
+ moveToElement(LoginLocator.LOGIN_BUTTON_MOVE);
+
+ // Whether to enter the specified page after login
+ return ifTitleContains(LoginData.TENANT);
+ }
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java
new file mode 100644
index 0000000000..8dd1010a82
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.page.project;
+
+import org.apache.dolphinscheduler.common.PageCommon;
+import org.apache.dolphinscheduler.constant.TestConstant;
+import org.apache.dolphinscheduler.data.project.CreatProjectData;
+import org.apache.dolphinscheduler.locator.project.CreateProjectLocator;
+import org.openqa.selenium.WebDriver;
+
+public class CreateProjectPage extends PageCommon {
+ public CreateProjectPage(WebDriver driver) {
+ super(driver);
+ }
+ /**
+ * jump page
+ */
+ public void jumpProjectManagePage() throws InterruptedException {
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+ clickElement(CreateProjectLocator.PROJECT_MANAGE);
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+ }
+
+ /**
+ * creatTenant
+ *
+ * @return Whether to enter the specified page after creat tenant
+ */
+ public boolean createProject() throws InterruptedException {
+ //click create project
+ clickElement(CreateProjectLocator.CREATE_PROJECT_BUTTON);
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+
+ // input create project data
+ sendInput(CreateProjectLocator.PROJECT_NAME, CreatProjectData.PROJECT_NAME);
+ sendInput(CreateProjectLocator.PROJECT_DESCRIPTION, CreatProjectData.DESCRIPTION);
+
+ // click submit button
+ clickButton(CreateProjectLocator.SUBMIT_BUTTON);
+
+ // Whether to enter the specified page after submit
+ return ifTitleContains(CreatProjectData.PROJECT_TITLE);
+ }
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java
new file mode 100644
index 0000000000..aeec83cd19
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.page.project;
+
+import org.apache.dolphinscheduler.common.PageCommon;
+import org.apache.dolphinscheduler.constant.TestConstant;
+import org.apache.dolphinscheduler.data.project.CreatWorkflowData;
+import org.apache.dolphinscheduler.locator.project.CreateWorkflowLocator;
+import org.openqa.selenium.WebDriver;
+
+public class CreateWorkflowPage extends PageCommon {
+ public CreateWorkflowPage(WebDriver driver) {
+ super(driver);
+ }
+ /**
+ * jump page
+ */
+
+ public boolean createWorkflow() throws InterruptedException {
+ // click project name
+ clickElement(CreateWorkflowLocator.CLICK_PROJECT_NAME);
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+
+
+ // click workflow define
+ clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_DEFINE);
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+
+ // click create workflow button
+ clickElement(CreateWorkflowLocator.CLICK_CREATE_WORKFLOW_BUTTON);
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+
+ //drag shell_task
+ dragAndDrop(CreateWorkflowLocator.MOUSE_DOWN_AT_SHELL,CreateWorkflowLocator.MOUSE_MOVE_SHELL_AT_DAG);
+
+ //input shell task _name
+ sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_NAME , CreatWorkflowData.SHELL_TASK_NAME);
+
+ //click stop run type
+ clickElement(CreateWorkflowLocator.CLICK_STOP_RUN_TYPE);
+
+ //click normal run type
+ clickElement(CreateWorkflowLocator.CLICK_NORMAL_RUN_TYPE);
+
+ //input shell task description
+ sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_DESCRIPTION , CreatWorkflowData.SHELL_TASK_DESCRIPTION);
+
+ //select task priority
+ clickElement(CreateWorkflowLocator.CLICK_TASK_PRIORITY);
+ clickElement(CreateWorkflowLocator.SELECT_TASK_PRIORITY);
+
+ //select work group
+ clickElement(CreateWorkflowLocator.CLICK_WORK_GROUP);
+ clickElement(CreateWorkflowLocator.SELECT_WORK_GROUP);
+
+ //select number of failed retries
+ clickElement(CreateWorkflowLocator.SELECT_FAIL_RETRIES_NUMBER);
+
+ //select failed retry interval
+ clickElement(CreateWorkflowLocator.SELECT_FAIL_RETRIES_INTERVAL);
+
+
+ //click timeout alarm
+ clickElement(CreateWorkflowLocator.CLICK_TIMEOUT_ALARM);
+
+
+ //select timeout fail
+ clickElement(CreateWorkflowLocator.SELECT_TIMEOUT_FAIL);
+
+
+ //cancel timeout alarm
+ clickElement(CreateWorkflowLocator.CANCEL_TIMEOUT_ALARM);
+
+
+ //select timeout alarm
+ clickElement(CreateWorkflowLocator.SELECT_TIMEOUT_ALARM);
+
+ //clear timeout
+ clearInput(CreateWorkflowLocator.SELECT_TIMEOUT);
+ clearInput(CreateWorkflowLocator.SELECT_TIMEOUT);
+
+ //input timeout
+ sendInput(CreateWorkflowLocator.SELECT_TIMEOUT,CreatWorkflowData.INPUT_TIMEOUT);
+
+ //click codeMirror and input script
+ inputCodeMirror(CreateWorkflowLocator.CLICK_CODE_MIRROR, CreateWorkflowLocator.INPUT_SCRIPT,CreatWorkflowData.SHELL_SCRIPT);
+ scrollToElementBottom();
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+
+ //click custom parameters
+ clickElement(CreateWorkflowLocator.CLICK_CUSTOM_PARAMETERS);
+
+ //input custom parameters
+ sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS, CreatWorkflowData.INPUT_CUSTOM_PARAMETERS);
+
+ //input custom parameters value
+ sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS_VALUE, CreatWorkflowData.INPUT_CUSTOM_PARAMETERS_VALUE);
+
+ //click add custom parameters
+ clickElement(CreateWorkflowLocator.CLICK_ADD_CUSTOM_PARAMETERS);
+
+ scrollToElementBottom();
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+
+ //input add custom parameters
+ sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS,CreatWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS);
+
+ //input add custom parameters value
+ sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS_VALUE,CreatWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS_VALUE);
+
+ //click delete custom parameters
+ clickElement(CreateWorkflowLocator.CLICK_DELETE_CUSTOM_PARAMETERS);
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+
+ //click submit button
+ clickElement(CreateWorkflowLocator.CLICK_SUBMIT_BUTTON);
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+
+ moveToDragElement(CreateWorkflowLocator.MOUSE_MOVE_SHELL_AT_DAG,-300,-100);
+
+ return ifTitleContains(CreatWorkflowData.WORKFLOW_TITLE);
+ }
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java
new file mode 100644
index 0000000000..4c88f6575e
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.page.security;
+
+import org.apache.dolphinscheduler.common.PageCommon;
+import org.apache.dolphinscheduler.constant.TestConstant;
+import org.apache.dolphinscheduler.data.LoginData;
+import org.apache.dolphinscheduler.data.security.TenantManageData;
+import org.apache.dolphinscheduler.locator.LoginLocator;
+import org.apache.dolphinscheduler.locator.security.TenantManageLocator;
+import org.apache.dolphinscheduler.util.RedisUtil;
+import org.openqa.selenium.WebDriver;
+
+public class TenantManagePage extends PageCommon {
+ /**
+ * Unique constructor
+ * @param driver driver
+ */
+ public TenantManagePage(WebDriver driver) {
+ super(driver);
+ }
+
+
+ /**
+ * jump page
+ */
+ public void jumpPage() {
+ System.out.println("jump tenant page");
+ super.jumpPage(TenantManageData.TENANAT_URL);
+ }
+
+ /**
+ * createTenant
+ *
+ * @return Whether to enter the specified page after creat tenant
+ */
+ public boolean createTenant() throws InterruptedException {
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+
+ //create tenant
+ clickButton(TenantManageLocator.CREATE_TENANT_BUTTON);
+
+ // tenant data
+ sendInput(TenantManageLocator.TENANT_INPUT_CODE, TenantManageData.TENANAT_CODE);
+ sendInput(TenantManageLocator.TENANT_INPUT_NAME, TenantManageData.TENANAT_NAME);
+ sendInput(TenantManageLocator.QUEUE, TenantManageData.QUEUE);
+ sendInput(TenantManageLocator.DESCRIPTION, TenantManageData.DESCRIPTION);
+
+ // click button
+ clickButton(TenantManageLocator.SUBMIT_BUTTON);
+
+ // Whether to enter the specified page after submit
+ return ifTitleContains(TenantManageData.TENANAT_MANAGE);
+ }
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java
new file mode 100644
index 0000000000..3e2be2d126
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.page.security;
+
+import org.apache.dolphinscheduler.common.PageCommon;
+import org.apache.dolphinscheduler.constant.TestConstant;
+import org.apache.dolphinscheduler.data.security.UserManageData;
+import org.apache.dolphinscheduler.locator.security.UserManageLocator;
+import org.openqa.selenium.WebDriver;
+
+public class UserManagePage extends PageCommon {
+ public UserManagePage(WebDriver driver) {
+ super(driver);
+ }
+ /**
+ * jump page
+ */
+ public void jumpPage() {
+ System.out.println("jump tenant page");
+ super.jumpPage(UserManageData.USER_URL);
+ }
+
+ /**
+ * creatTenant
+ *
+ * @return Whether to enter the specified page after creat tenant
+ */
+ public boolean createUser() throws InterruptedException {
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+ // click user manage
+ clickElement(UserManageLocator.CLICK_USER_MANAGE);
+ Thread.sleep(TestConstant.ONE_THOUSANG);
+
+ // click create user button
+ clickButton(UserManageLocator.CLICK_CREATE_USER_BUTTON);
+
+ // input user data
+ sendInput(UserManageLocator.INPUT_USERNAME, UserManageData.USERNAME);
+ sendInput(UserManageLocator.INPUT_PASSWORD, UserManageData.PASSWORD);
+ clickButton(UserManageLocator.CLICK_TENANT);
+ clickButton(UserManageLocator.SELECT_TENANT);
+ clickButton(UserManageLocator.CLICK_QUEUE);
+ clickButton(UserManageLocator.SELECT_QUEUE);
+ sendInput(UserManageLocator.TENANT_INPUT_EMAIL, UserManageData.EMAIL);
+ sendInput(UserManageLocator.TENANT_INPUT_PHONE, UserManageData.PHONE);
+
+ // click button
+ clickButton(UserManageLocator.SUBMIT);
+
+ // Whether to enter the specified page after submit
+ return ifTitleContains(UserManageData.USER_MANAGE);
+ }
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java
new file mode 100644
index 0000000000..bd3c31d38b
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.testcase;
+
+import org.apache.dolphinscheduler.base.BaseTest;
+import org.apache.dolphinscheduler.page.LoginPage;
+import org.testng.annotations.Test;
+
+
+public class LoginTest extends BaseTest {
+ private LoginPage loginPage;
+ @Test(description = "LoginTest", priority = 1)
+ public void testLogin() throws InterruptedException {
+ // init login page
+ loginPage = new LoginPage(driver);
+
+ // enter login page
+ loginPage.jumpPage();
+
+ //assert login page
+ assert loginPage.login();
+ }
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateProjectTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateProjectTest.java
new file mode 100644
index 0000000000..8abd09d37c
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateProjectTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.testcase.project;
+
+import org.apache.dolphinscheduler.base.BaseTest;
+import org.apache.dolphinscheduler.page.project.CreateProjectPage;
+import org.testng.annotations.Test;
+
+public class CreateProjectTest extends BaseTest {
+ private CreateProjectPage createProjectPage;
+
+ @Test(description = "TenantTest", priority = 1)
+ public void testUserManage() throws InterruptedException {
+ createProjectPage = new CreateProjectPage(driver);
+ // enter user manage page
+ createProjectPage.jumpProjectManagePage();
+ //assert user manage page
+ assert createProjectPage.createProject();
+ }
+
+}
+
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateWorkflowTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateWorkflowTest.java
new file mode 100644
index 0000000000..6ac13f8124
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateWorkflowTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.testcase.project;
+
+import org.apache.dolphinscheduler.base.BaseTest;
+import org.apache.dolphinscheduler.page.project.CreateProjectPage;
+import org.apache.dolphinscheduler.page.project.CreateWorkflowPage;
+import org.testng.annotations.Test;
+
+public class CreateWorkflowTest extends BaseTest {
+ private CreateWorkflowPage createWorkflowPage;
+ private CreateProjectPage createProjectPage;
+
+
+ @Test(description = "CreateWorkflowTest", priority = 1)
+ public void CreateWorkflowTest() throws InterruptedException {
+ createProjectPage = new CreateProjectPage(driver);
+ createProjectPage.jumpProjectManagePage();
+ createWorkflowPage = new CreateWorkflowPage(driver);
+ //assert create workflow
+ assert createWorkflowPage.createWorkflow();
+ }
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/TenantManageTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/TenantManageTest.java
new file mode 100644
index 0000000000..7124b4e094
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/TenantManageTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.testcase.security;
+
+import org.apache.dolphinscheduler.base.BaseTest;
+import org.apache.dolphinscheduler.page.security.TenantManagePage;
+import org.testng.annotations.Test;
+
+public class TenantManageTest extends BaseTest {
+ private TenantManagePage tenantManagePage;
+
+ @Test(description = "TenantTest", priority = 1)
+ public void testTenantManage() throws InterruptedException {
+ tenantManagePage = new TenantManagePage(driver);
+ // enter tenant manage page
+ tenantManagePage.jumpPage();
+ //assert tenant manage page
+ assert tenantManagePage.createTenant();
+ }
+}
diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/UserManageTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/UserManageTest.java
new file mode 100644
index 0000000000..834ebdbbf0
--- /dev/null
+++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/UserManageTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.testcase.security;
+
+import org.apache.dolphinscheduler.base.BaseTest;
+import org.apache.dolphinscheduler.page.security.UserManagePage;
+import org.testng.annotations.Test;
+
+public class UserManageTest extends BaseTest {
+ private UserManagePage userManagePage;
+
+ @Test(description = "TenantTest", priority = 1)
+ public void testUserManage() throws InterruptedException {
+ userManagePage = new UserManagePage(driver);
+ // enter user manage page
+ userManagePage.jumpPage();
+ //assert user manage page
+ assert userManagePage.createUser();
+ }
+}
diff --git a/e2e/src/test/resources/config/config.properties b/e2e/src/test/resources/config/config.properties
new file mode 100644
index 0000000000..6a01234d01
--- /dev/null
+++ b/e2e/src/test/resources/config/config.properties
@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+############### project ##############
+# login url
+LOGIN_URL=http://127.0.0.1:8888/dolphinscheduler/
+#login username
+USER_NAME=admin
+#login password
+PASSWORD=dolphinscheduler123
+
+############### web driver ##############
+# driver path
+driver.chromeDriver=/usr/local/bin/chromedriver
+# implicitly wait(s)
+driver.timeouts.implicitlyWait=10
+# show wait(s)
+driver.timeouts.webDriverWait=10
+# page load timeout(s)
+driver.timeouts.pageLoadTimeout=10
+# JS wait timeouts(s)
+driver.timeouts.setScriptTimeout=10
+
+
+############### redis ##############
+# redis ip
+redis.ip=127.0.0.1
+# redis port
+redis.port=6379
+# redis password
+redis.pwd=
+############### redis pool ##############
+# jedis expireTime(s)
+jedis.expireTime=3600
+# jedis maxTotal
+jedis.pool.maxTotal=3000
+# jedis maxIdle
+jedis.pool.maxIdle=1000
+# jedis maxWaitMillis
+jedis.pool.maxWaitMillis=10000
+# jedis Whether to perform a valid check when calling the borrowObject method
+jedis.pool.testOnBorrow=true
+# jedis Whether to perform a valid check when calling the returnObject method
+jedis.pool.testOnReturn=true
diff --git a/e2e/suite.xml b/e2e/suite.xml
new file mode 100644
index 0000000000..d9d7ae6845
--- /dev/null
+++ b/e2e/suite.xml
@@ -0,0 +1,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/e2e/testng.xml b/e2e/testng.xml
new file mode 100644
index 0000000000..757ffab248
--- /dev/null
+++ b/e2e/testng.xml
@@ -0,0 +1,22 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/install.sh b/install.sh
index 354cdd2be3..8735543c47 100644
--- a/install.sh
+++ b/install.sh
@@ -422,17 +422,8 @@ fi
done
-# 3,stop server
-echo "3,stop server"
-sh ${workDir}/script/stop-all.sh
-
-# 4,delete zk node
-echo "4,delete zk node"
-
-sh ${workDir}/script/remove-zk-node.sh $zkRoot
-
-# 5,scp resources
-echo "5,scp resources"
+# 3,scp resources
+echo "3,scp resources"
sh ${workDir}/script/scp-hosts.sh
if [ $? -eq 0 ]
then
@@ -442,6 +433,18 @@ else
exit -1
fi
+
+# 4,stop server
+echo "4,stop server"
+sh ${workDir}/script/stop-all.sh
+
+
+# 5,delete zk node
+echo "5,delete zk node"
+
+sh ${workDir}/script/remove-zk-node.sh $zkRoot
+
+
# 6,startup
echo "6,startup"
sh ${workDir}/script/start-all.sh
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 53184e7412..8a1e9b98f2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -229,7 +229,12 @@
org.apache.dolphinscheduler
- dolphinscheduler-rpc
+ dolphinscheduler-remote
+ ${project.version}
+
+
+ org.apache.dolphinscheduler
+ dolphinscheduler-service
${project.version}
@@ -237,7 +242,6 @@
dolphinscheduler-alert
${project.version}
-
org.apache.curator
curator-framework
@@ -680,8 +684,10 @@
**/common/threadutils/*.java
**/common/graph/*.java
**/common/queue/*.java
+ **/common/task/SqoopParameterEntityTest.java
**/api/utils/CheckUtilsTest.java
**/api/utils/FileUtilsTest.java
+ **/api/utils/FourLetterWordTest.java
**/api/utils/exportprocess/DataSourceParamTest.java
**/api/utils/exportprocess/DependentParamTest.java
**/api/enums/*.java
@@ -702,6 +708,8 @@
**/api/service/BaseDAGServiceTest.java
**/api/service/LoggerServiceTest.java
**/api/service/DataAnalysisServiceTest.java
+ **/api/service/ProcessInstanceServiceTest.java
+ **/api/service/TaskInstanceServiceTest.java
**/alert/utils/ExcelUtilsTest.java
**/alert/utils/FuncUtilsTest.java
**/alert/utils/JSONUtilsTest.java
@@ -714,10 +722,13 @@
**/dao/mapper/AlertGroupMapperTest.java
**/dao/mapper/AlertMapperTest.java
**/dao/mapper/CommandMapperTest.java
+ **/dao/entity/TaskInstanceTest.java
**/dao/cron/CronUtilsTest.java
+ **/dao/utils/DagHelperTest.java
**/alert/template/AlertTemplateFactoryTest.java
**/alert/template/impl/DefaultHTMLTemplateTest.java
**/server/worker/task/datax/DataxTaskTest.java
+ **/server/worker/task/sqoop/SqoopTaskTest.java
**/server/utils/DataxUtilsTest.java
@@ -773,9 +784,6 @@
**/dolphinscheduler-ui/src/view/common/outro.inc
**/dolphinscheduler-ui/src/view/common/meta.inc
**/dolphinscheduler-ui/src/combo/1.0.0/3rd.css
-
- **/dolphinscheduler-rpc/src/main/java/org/apache/dolphinscheduler/rpc/LogViewServiceGrpc.java
-
true
@@ -862,8 +870,9 @@
dolphinscheduler-api
dolphinscheduler-dao
dolphinscheduler-alert
- dolphinscheduler-rpc
dolphinscheduler-dist
+ dolphinscheduler-remote
+ dolphinscheduler-service
diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh
index d4db103fe1..3e7f10783a 100644
--- a/script/dolphinscheduler-daemon.sh
+++ b/script/dolphinscheduler-daemon.sh
@@ -44,7 +44,7 @@ export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/logs
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
-export DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms4g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
+export DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
export STOP_TIMEOUT=5
if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then
@@ -69,10 +69,7 @@ elif [ "$command" = "alert-server" ]; then
LOG_FILE="-Dserver=alert-server"
CLASS=org.apache.dolphinscheduler.alert.AlertServer
elif [ "$command" = "logger-server" ]; then
- CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer
-elif [ "$command" = "combined-server" ]; then
- LOG_FILE="-Dlogging.config=classpath:combined_logback.xml -Dspring.profiles.active=api -Dserver.is-combined-server=true"
- CLASS=org.apache.dolphinscheduler.api.CombinedApplicationServer
+ CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
else
echo "Error: No command named \`$command' was found."
exit 1
diff --git a/sql/dolphinscheduler-postgre.sql b/sql/dolphinscheduler-postgre.sql
index c68fd17be1..88d62737de 100644
--- a/sql/dolphinscheduler-postgre.sql
+++ b/sql/dolphinscheduler-postgre.sql
@@ -574,6 +574,7 @@ CREATE TABLE t_ds_task_instance (
max_retry_times int DEFAULT NULL ,
task_instance_priority int DEFAULT NULL ,
worker_group_id int DEFAULT '-1' ,
+ executor_id int DEFAULT NULL ,
PRIMARY KEY (id)
) ;
diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql
index ea0f9cb022..68393702fc 100644
--- a/sql/dolphinscheduler_mysql.sql
+++ b/sql/dolphinscheduler_mysql.sql
@@ -616,6 +616,7 @@ CREATE TABLE `t_ds_task_instance` (
`max_retry_times` int(2) DEFAULT NULL COMMENT 'max retry times',
`task_instance_priority` int(11) DEFAULT NULL COMMENT 'task instance priority:0 Highest,1 High,2 Medium,3 Low,4 Lowest',
`worker_group_id` int(11) DEFAULT '-1' COMMENT 'worker group id',
+ `executor_id` int(11) DEFAULT NULL COMMENT 'executor id',
PRIMARY KEY (`id`),
KEY `process_instance_id` (`process_instance_id`) USING BTREE,
KEY `task_instance_index` (`process_definition_id`,`process_instance_id`) USING BTREE,
diff --git a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql
index 9fe246a8c2..790a9a8ada 100644
--- a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql
+++ b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql
@@ -35,3 +35,23 @@ d//
delimiter ;
CALL uc_dolphin_T_t_ds_process_definition_A_modify_by;
DROP PROCEDURE uc_dolphin_T_t_ds_process_definition_A_modify_by;
+
+-- uc_dolphin_T_t_ds_process_definition_A_modify_by
+drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_instance_A_executor_id;
+delimiter d//
+CREATE PROCEDURE uc_dolphin_T_t_ds_task_instance_A_executor_id()
+ BEGIN
+ IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
+ WHERE TABLE_NAME='t_ds_task_instance'
+ AND TABLE_SCHEMA=(SELECT DATABASE())
+ AND COLUMN_NAME ='executor_id')
+ THEN
+ ALTER TABLE t_ds_task_instance ADD `executor_id` int(11) DEFAULT NULL COMMENT 'executor id';
+ END IF;
+ END;
+
+d//
+
+delimiter ;
+CALL uc_dolphin_T_t_ds_task_instance_A_executor_id;
+DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_A_executor_id;
\ No newline at end of file
diff --git a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql
index 7fc12900e4..cbe7c22bbe 100644
--- a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql
+++ b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql
@@ -32,3 +32,20 @@ delimiter ;
SELECT uc_dolphin_T_t_ds_process_definition_A_modify_by();
DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_process_definition_A_modify_by();
+-- uc_dolphin_T_t_ds_process_definition_A_modify_by
+delimiter d//
+CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_task_instance_A_executor_id() RETURNS void AS $$
+BEGIN
+ IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS
+ WHERE TABLE_NAME='t_ds_task_instance'
+ AND COLUMN_NAME ='executor_id')
+ THEN
+ ALTER TABLE t_ds_task_instance ADD COLUMN executor_id int DEFAULT NULL;
+ END IF;
+END;
+$$ LANGUAGE plpgsql;
+d//
+
+delimiter ;
+SELECT uc_dolphin_T_t_ds_task_instance_A_executor_id();
+DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_A_executor_id();
|