github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[Fix-5795][Improvement][Server] The starttime field in the HttpTask log is not displayed as expected. (#5796) 

* improve timestamp format

make the startime in the log of httptask to be easier to read.


* fix bad code smell and update the note.
2.0.7-release
kyoty 3 years ago committed by GitHub
parent
16986c3c65
commit
7bffe0ac85
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 113 additions and 59 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 52
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
  2. 15
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java
  3. 105
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java

52
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
Unescape View File

@ -26,6 +26,7 @@ import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Calendar;
import java.util.Date;
import java.util.Objects;
import java.util.TimeZone;
import org.slf4j.Logger;
@ -36,12 +37,44 @@ import org.slf4j.LoggerFactory;
*/
public class DateUtils {
static final long C0 = 1L;
static final long C1 = C0 * 1000L;
static final long C2 = C1 * 1000L;
static final long C3 = C2 * 1000L;
static final long C4 = C3 * 60L;
static final long C5 = C4 * 60L;
static final long C6 = C5 * 24L;
/**
* a default datetime formatter for the timestamp
*/
private static final DateTimeFormatter DEFAULT_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
private static final Logger logger = LoggerFactory.getLogger(DateUtils.class);
private DateUtils() {
throw new UnsupportedOperationException("Construct DateUtils");
}
/**
* @param timeMillis timeMillis like System.currentTimeMillis()
* @return string formatted as yyyy-MM-dd HH:mm:ss
*/
public static String formatTimeStamp(long timeMillis) {
return formatTimeStamp(timeMillis, DEFAULT_DATETIME_FORMATTER);
}
/**
* @param timeMillis timeMillis like System.currentTimeMillis()
* @param dateTimeFormatter expect formatter, like yyyy-MM-dd HH:mm:ss
* @return formatted string
*/
public static String formatTimeStamp(long timeMillis, DateTimeFormatter dateTimeFormatter) {
Objects.requireNonNull(dateTimeFormatter);
return dateTimeFormatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timeMillis),
ZoneId.systemDefault()));
}
/**
* date to local datetime
*
@ -253,7 +286,6 @@ public class DateUtils {
}
/**
*
* format time to duration
*
* @param d1 d1
@ -522,14 +554,6 @@ public class DateUtils {
return TimeZone.getTimeZone(timezoneId);
}
static final long C0 = 1L;
static final long C1 = C0 * 1000L;
static final long C2 = C1 * 1000L;
static final long C3 = C2 * 1000L;
static final long C4 = C3 * 60L;
static final long C5 = C4 * 60L;
static final long C6 = C5 * 24L;
/**
* Time unit representing one thousandth of a second
*/
@ -543,23 +567,23 @@ public class DateUtils {
return d / (C4 / C2);
}
public static long toHours(long d) {
public static long toHours(long d) {
return d / (C5 / C2);
}
public static long toDays(long d) {
public static long toDays(long d) {
return d / (C6 / C2);
}
public static long toDurationSeconds(long d) {
public static long toDurationSeconds(long d) {
return (d % (C4 / C2)) / (C3 / C2);
}
public static long toDurationMinutes(long d) {
public static long toDurationMinutes(long d) {
return (d % (C5 / C2)) / (C4 / C2);
}
public static long toDurationHours(long d) {
public static long toDurationHours(long d) {
return (d % (C6 / C2)) / (C5 / C2);
}

15
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java
Unescape View File

@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.utils;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.format.DateTimeFormatter;
import java.util.Date;
import java.util.TimeZone;
@ -41,6 +42,20 @@ public class DateUtilsTest {
Assert.assertEquals("01 09:23:08", readableDate);
}
@Test
public void testConvertTimeStampsToString() {
TimeZone defaultTimeZone = TimeZone.getDefault();
final TimeZone timeZone = TimeZone.getTimeZone("Asia/Shanghai");
TimeZone.setDefault(timeZone);
long timeMillis = 1625989249021L;
Assert.assertEquals("2021-07-11 15:40:49", DateUtils.formatTimeStamp(timeMillis));
DateTimeFormatter testFormatter = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss");
Assert.assertEquals("2021/07/11 15:40:49", DateUtils.formatTimeStamp(timeMillis, testFormatter));
TimeZone.setDefault(defaultTimeZone);
}
@Test
public void testWeek() {
Date curr = DateUtils.stringToDate("2019-02-01 00:00:00");

105
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java
Unescape View File

@ -14,11 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.worker.task.http;
package org.apache.dolphinscheduler.server.worker.task.http;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.io.Charsets;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.HttpMethod;
@ -27,10 +25,16 @@ import org.apache.dolphinscheduler.common.process.HttpProperty;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.http.HttpParameters;
import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.commons.io.Charsets;
import org.apache.http.HttpEntity;
import org.apache.http.ParseException;
import org.apache.http.client.config.RequestConfig;
@ -42,7 +46,6 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@ -50,28 +53,27 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import com.fasterxml.jackson.databind.node.ObjectNode;
/**
* http task
*/
public class HttpTask extends AbstractTask {
/**
* http parameters
*/
private HttpParameters httpParameters;
/**
* application json
*/
protected static final String APPLICATION_JSON = "application/json";
/**
* output
*/
protected String output;
/**
* http parameters
*/
private HttpParameters httpParameters;
/**
* taskExecutionContext
*/
@ -79,8 +81,9 @@ public class HttpTask extends AbstractTask {
/**
* constructor
* @param taskExecutionContext taskExecutionContext
* @param logger logger
*
* @param taskExecutionContext taskExecutionContext
* @param logger logger
*/
public HttpTask(TaskExecutionContext taskExecutionContext, Logger logger) {
super(taskExecutionContext, logger);
@ -103,27 +106,30 @@ public class HttpTask extends AbstractTask {
Thread.currentThread().setName(threadLoggerInfoName);
long startTime = System.currentTimeMillis();
String formatTimeStamp = DateUtils.formatTimeStamp(startTime);
String statusCode = null;
String body = null;
try(CloseableHttpClient client = createHttpClient();
CloseableHttpResponse response = sendRequest(client)) {
try (CloseableHttpClient client = createHttpClient();
CloseableHttpResponse response = sendRequest(client)) {
statusCode = String.valueOf(getStatusCode(response));
body = getResponseBody(response);
exitStatusCode = validResponse(body, statusCode);
long costTime = System.currentTimeMillis() - startTime;
logger.info("startTime: {}, httpUrl: {}, httpMethod: {}, costTime : {}Millisecond, statusCode : {}, body : {}, log : {}",
DateUtils.format2Readable(startTime), httpParameters.getUrl(),httpParameters.getHttpMethod(), costTime, statusCode, body, output);
}catch (Exception e){
logger.info("startTime: {}, httpUrl: {}, httpMethod: {}, costTime : {} milliseconds, statusCode : {}, body : {}, log : {}",
formatTimeStamp, httpParameters.getUrl(),
httpParameters.getHttpMethod(), costTime, statusCode, body, output);
} catch (Exception e) {
appendMessage(e.toString());
exitStatusCode = -1;
logger.error("httpUrl[" + httpParameters.getUrl() + "] connection failed："+output, e);
logger.error("httpUrl[" + httpParameters.getUrl() + "] connection failed：" + output, e);
throw e;
}
}
/**
* send request
*
* @param client client
* @return CloseableHttpResponse
* @throws IOException io exception
@ -139,23 +145,24 @@ public class HttpTask extends AbstractTask {
CommandType.of(taskExecutionContext.getCmdTypeIfComplement()),
taskExecutionContext.getScheduleTime());
List<HttpProperty> httpPropertyList = new ArrayList<>();
if(CollectionUtils.isNotEmpty(httpParameters.getHttpParams() )){
for (HttpProperty httpProperty: httpParameters.getHttpParams()) {
if (CollectionUtils.isNotEmpty(httpParameters.getHttpParams())) {
for (HttpProperty httpProperty : httpParameters.getHttpParams()) {
String jsonObject = JSONUtils.toJsonString(httpProperty);
String params = ParameterUtils.convertParameterPlaceholders(jsonObject,ParamUtils.convert(paramsMap));
logger.info("http request params：{}",params);
httpPropertyList.add(JSONUtils.parseObject(params,HttpProperty.class));
String params = ParameterUtils.convertParameterPlaceholders(jsonObject, ParamUtils.convert(paramsMap));
logger.info("http request params：{}", params);
httpPropertyList.add(JSONUtils.parseObject(params, HttpProperty.class));
}
}
addRequestParams(builder,httpPropertyList);
String requestUrl = ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(),ParamUtils.convert(paramsMap));
addRequestParams(builder, httpPropertyList);
String requestUrl = ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParamUtils.convert(paramsMap));
HttpUriRequest request = builder.setUri(requestUrl).build();
setHeaders(request,httpPropertyList);
setHeaders(request, httpPropertyList);
return client.execute(request);
}
/**
* get response body
*
* @param httpResponse http response
* @return response body
* @throws ParseException parse exception
@ -174,6 +181,7 @@ public class HttpTask extends AbstractTask {
/**
* get status code
*
* @param httpResponse http response
* @return status code
*/
@ -183,11 +191,12 @@ public class HttpTask extends AbstractTask {
/**
* valid response
* @param body body
* @param statusCode status code
*
* @param body body
* @param statusCode status code
* @return exit status code
*/
protected int validResponse(String body, String statusCode){
protected int validResponse(String body, String statusCode) {
int exitStatusCode = 0;
switch (httpParameters.getHttpCheckCondition()) {
case BODY_CONTAINS:
@ -226,6 +235,7 @@ public class HttpTask extends AbstractTask {
/**
* append message
*
* @param message message
*/
protected void appendMessage(String message) {
@ -239,17 +249,18 @@ public class HttpTask extends AbstractTask {
/**
* add request params
* @param builder buidler
* @param httpPropertyList http property list
*
* @param builder buidler
* @param httpPropertyList http property list
*/
protected void addRequestParams(RequestBuilder builder,List<HttpProperty> httpPropertyList) {
if(CollectionUtils.isNotEmpty(httpPropertyList)){
protected void addRequestParams(RequestBuilder builder, List<HttpProperty> httpPropertyList) {
if (CollectionUtils.isNotEmpty(httpPropertyList)) {
ObjectNode jsonParam = JSONUtils.createObjectNode();
for (HttpProperty property: httpPropertyList){
if(property.getHttpParametersType() != null){
if (property.getHttpParametersType().equals(HttpParametersType.PARAMETER)){
for (HttpProperty property : httpPropertyList) {
if (property.getHttpParametersType() != null) {
if (property.getHttpParametersType().equals(HttpParametersType.PARAMETER)) {
builder.addParameter(property.getProp(), property.getValue());
}else if(property.getHttpParametersType().equals(HttpParametersType.BODY)){
} else if (property.getHttpParametersType().equals(HttpParametersType.BODY)) {
jsonParam.put(property.getProp(), property.getValue());
}
}
@ -263,12 +274,13 @@ public class HttpTask extends AbstractTask {
/**
* set headers
* @param request request
* @param httpPropertyList http property list
*
* @param request request
* @param httpPropertyList http property list
*/
protected void setHeaders(HttpUriRequest request,List<HttpProperty> httpPropertyList) {
if(CollectionUtils.isNotEmpty(httpPropertyList)){
for (HttpProperty property: httpPropertyList) {
protected void setHeaders(HttpUriRequest request, List<HttpProperty> httpPropertyList) {
if (CollectionUtils.isNotEmpty(httpPropertyList)) {
for (HttpProperty property : httpPropertyList) {
if (HttpParametersType.HEADERS.equals(property.getHttpParametersType())) {
request.addHeader(property.getProp(), property.getValue());
}
@ -278,6 +290,7 @@ public class HttpTask extends AbstractTask {
/**
* create http client
*
* @return CloseableHttpClient
*/
protected CloseableHttpClient createHttpClient() {
@ -289,6 +302,7 @@ public class HttpTask extends AbstractTask {
/**
* request config
*
* @return RequestConfig
*/
private RequestConfig requestConfig() {
@ -297,6 +311,7 @@ public class HttpTask extends AbstractTask {
/**
* create request builder
*
* @return RequestBuilder
*/
protected RequestBuilder createRequestBuilder() {

Write Preview
Loading…
Cancel
Save