Browse Source

Merge pull request #5 from apache/dev

merge
pull/2/head
Tboy 5 years ago committed by GitHub
parent
commit
35b6442fd9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      .github/workflows/ci_ut.yml
  2. 7
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java
  3. 2
      dolphinscheduler-alert/src/main/resources/mail_templates/alert_mail_template.ftl
  4. 6
      dolphinscheduler-dao/src/main/resources/application-dao.properties
  5. 75
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java
  6. 131
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java
  7. 2
      pom.xml

2
.github/workflows/ci_ut.yml

@ -39,7 +39,7 @@ jobs:
restore-keys: | restore-keys: |
${{ runner.os }}-maven- ${{ runner.os }}-maven-
- name: Bootstrap database - name: Bootstrap database
run: cd ${DOCKER_DIR} && docker-compose up -d db run: cd ${DOCKER_DIR} && docker-compose up -d
- name: Set up JDK 1.8 - name: Set up JDK 1.8
uses: actions/setup-java@v1 uses: actions/setup-java@v1
with: with:

7
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java

@ -20,6 +20,8 @@ import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow; import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.HorizontalAlignment;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -71,10 +73,14 @@ public class ExcelUtils {
//set the height of the first line //set the height of the first line
row.setHeight((short)500); row.setHeight((short)500);
//set Horizontal right
CellStyle cellStyle = wb.createCellStyle();
cellStyle.setAlignment(HorizontalAlignment.RIGHT);
//setting excel headers //setting excel headers
for (int i = 0; i < headerList.size(); i++) { for (int i = 0; i < headerList.size(); i++) {
HSSFCell cell = row.createCell(i); HSSFCell cell = row.createCell(i);
cell.setCellStyle(cellStyle);
cell.setCellValue(headerList.get(i)); cell.setCellValue(headerList.get(i));
} }
@ -88,6 +94,7 @@ public class ExcelUtils {
rowIndex++; rowIndex++;
for (int j = 0 ; j < values.length ; j++){ for (int j = 0 ; j < values.length ; j++){
HSSFCell cell1 = row.createCell(j); HSSFCell cell1 = row.createCell(j);
cell1.setCellStyle(cellStyle);
cell1.setCellValue(String.valueOf(values[j])); cell1.setCellValue(String.valueOf(values[j]));
} }
} }

2
dolphinscheduler-alert/src/main/resources/mail_templates/alert_mail_template.ftl

@ -14,4 +14,4 @@
~ See the License for the specific language governing permissions and ~ See the License for the specific language governing permissions and
~ limitations under the License. ~ limitations under the License.
--> -->
<!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN' 'http://www.w3.org/TR/html4/loose.dtd'><html><head><title> dolphinscheduler</title><meta name='Keywords' content=''><meta name='Description' content=''><style type="text/css">table { margin-top:0px; padding-top:0px; border:1px solid; font-size: 14px; color: #333333; border-width: 1px; border-color: #666666; border-collapse: collapse; } table th { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #dedede; } table td { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #ffffff; }</style></head><body style="margin:0;padding:0"><table border="1px" cellpadding="5px" cellspacing="-10px"><thead><#if title??> ${title}</#if></thead><#if content??> ${content}</#if></table></body></html> <!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN' 'http://www.w3.org/TR/html4/loose.dtd'><html><head><title> dolphinscheduler</title><meta name='Keywords' content=''><meta name='Description' content=''><style type="text/css">table { margin-top:0px; padding-top:0px; border:1px solid; font-size: 14px; color: #333333; border-width: 1px; border-color: #666666; border-collapse: collapse; } table th { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #dedede; text-align: right;} table td { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #ffffff; text-align: right;}</style></head><body style="margin:0;padding:0"><table border="1px" cellpadding="5px" cellspacing="-10px"><thead><#if title??> ${title}</#if></thead><#if content??> ${content}</#if></table></body></html>

6
dolphinscheduler-dao/src/main/resources/application-dao.properties

@ -19,12 +19,12 @@
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
# postgre # postgre
spring.datasource.driver-class-name=org.postgresql.Driver spring.datasource.driver-class-name=org.postgresql.Driver
spring.datasource.url=jdbc:postgresql://192.168.xx.xx:5432/dolphinscheduler spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler
# mysql # mysql
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
spring.datasource.username=xx spring.datasource.username=test
spring.datasource.password=xx spring.datasource.password=test
# connection configuration # connection configuration
spring.datasource.initialSize=5 spring.datasource.initialSize=5

75
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java

@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -44,9 +45,11 @@ public class FlinkArgsUtils {
*/ */
public static List<String> buildArgs(FlinkParameters param) { public static List<String> buildArgs(FlinkParameters param) {
List<String> args = new ArrayList<>(); List<String> args = new ArrayList<>();
String deployMode = "cluster"; String deployMode = "cluster";
if (StringUtils.isNotEmpty(param.getDeployMode())) { String tmpDeployMode = param.getDeployMode();
deployMode = param.getDeployMode(); if (StringUtils.isNotEmpty(tmpDeployMode)) {
deployMode = tmpDeployMode;
} }
if (!"local".equals(deployMode)) { if (!"local".equals(deployMode)) {
@ -54,68 +57,70 @@ public class FlinkArgsUtils {
args.add(Constants.FLINK_YARN_CLUSTER); //yarn-cluster args.add(Constants.FLINK_YARN_CLUSTER); //yarn-cluster
int slot = param.getSlot();
if (param.getSlot() != 0) { if (slot != 0) {
args.add(Constants.FLINK_YARN_SLOT); args.add(Constants.FLINK_YARN_SLOT);
args.add(String.format("%d", param.getSlot())); //-ys args.add(String.format("%d", slot)); //-ys
} }
if (StringUtils.isNotEmpty(param.getAppName())) { //-ynm String appName = param.getAppName();
if (StringUtils.isNotEmpty(appName)) { //-ynm
args.add(Constants.FLINK_APP_NAME); args.add(Constants.FLINK_APP_NAME);
args.add(param.getAppName()); args.add(appName);
} }
if (param.getTaskManager() != 0) { //-yn int taskManager = param.getTaskManager();
if (taskManager != 0) { //-yn
args.add(Constants.FLINK_TASK_MANAGE); args.add(Constants.FLINK_TASK_MANAGE);
args.add(String.format("%d", param.getTaskManager())); args.add(String.format("%d", taskManager));
} }
if (StringUtils.isNotEmpty(param.getJobManagerMemory())) { String jobManagerMemory = param.getJobManagerMemory();
if (StringUtils.isNotEmpty(jobManagerMemory)) {
args.add(Constants.FLINK_JOB_MANAGE_MEM); args.add(Constants.FLINK_JOB_MANAGE_MEM);
args.add(param.getJobManagerMemory()); //-yjm args.add(jobManagerMemory); //-yjm
} }
if (StringUtils.isNotEmpty(param.getTaskManagerMemory())) { // -ytm String taskManagerMemory = param.getTaskManagerMemory();
if (StringUtils.isNotEmpty(taskManagerMemory)) { // -ytm
args.add(Constants.FLINK_TASK_MANAGE_MEM); args.add(Constants.FLINK_TASK_MANAGE_MEM);
args.add(param.getTaskManagerMemory()); args.add(taskManagerMemory);
} }
args.add(Constants.FLINK_detach); //-d args.add(Constants.FLINK_detach); //-d
} }
if (param.getProgramType() != null) { ProgramType programType = param.getProgramType();
if (param.getProgramType() != ProgramType.PYTHON) { String mainClass = param.getMainClass();
if (StringUtils.isNotEmpty(param.getMainClass())) { if (programType != null && programType != ProgramType.PYTHON && StringUtils.isNotEmpty(mainClass)) {
args.add(Constants.FLINK_MAIN_CLASS); //-c args.add(Constants.FLINK_MAIN_CLASS); //-c
args.add(param.getMainClass()); //main class args.add(param.getMainClass()); //main class
}
}
} }
if (param.getMainJar() != null) { ResourceInfo mainJar = param.getMainJar();
args.add(param.getMainJar().getRes()); if (mainJar != null) {
args.add(mainJar.getRes());
} }
if (StringUtils.isNotEmpty(param.getMainArgs())) { String mainArgs = param.getMainArgs();
args.add(param.getMainArgs()); if (StringUtils.isNotEmpty(mainArgs)) {
args.add(mainArgs);
} }
// --files --conf --libjar ... // --files --conf --libjar ...
if (StringUtils.isNotEmpty(param.getOthers())) { String others = param.getOthers();
String others = param.getOthers(); String queue = param.getQueue();
if (!others.contains("--qu")) { if (StringUtils.isNotEmpty(others)) {
if (StringUtils.isNotEmpty(param.getQueue()) && !deployMode.equals("local")) {
args.add(Constants.FLINK_QUEUE); if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) {
args.add(param.getQueue()); args.add(Constants.FLINK_QUEUE);
} args.add(param.getQueue());
} }
args.add(param.getOthers()); args.add(others);
} else if (StringUtils.isNotEmpty(param.getQueue()) && !deployMode.equals("local")) { } else if (StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) {
args.add(Constants.FLINK_QUEUE); args.add(Constants.FLINK_QUEUE);
args.add(param.getQueue()); args.add(param.getQueue());
} }
return args; return args;

131
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java

@ -0,0 +1,131 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
/**
* Test FlinkArgsUtils
*/
public class FlinkArgsUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(FlinkArgsUtilsTest.class);
public String mode = "cluster";
public int slot = 2;
public String appName = "testFlink";
public int taskManager = 4;
public String taskManagerMemory = "2G";
public String jobManagerMemory = "4G";
public ProgramType programType = ProgramType.JAVA;
public String mainClass = "com.test";
public ResourceInfo mainJar = null;
public String mainArgs = "testArgs";
public String queue = "queue1";
public String others = "--input file:///home";
@Before
public void setUp() throws Exception {
ResourceInfo main = new ResourceInfo();
main.setRes("testflink-1.0.0-SNAPSHOT.jar");
mainJar = main;
}
/**
* Test buildArgs
*/
@Test
public void testBuildArgs() {
//Define params
FlinkParameters param = new FlinkParameters();
param.setDeployMode(mode);
param.setMainClass(mainClass);
param.setAppName(appName);
param.setSlot(slot);
param.setTaskManager(taskManager);
param.setJobManagerMemory(jobManagerMemory);
param.setTaskManagerMemory(taskManagerMemory);
param.setMainJar(mainJar);
param.setProgramType(programType);
param.setMainArgs(mainArgs);
param.setQueue(queue);
param.setOthers(others);
//Invoke buildArgs
List<String> result = FlinkArgsUtils.buildArgs(param);
for (String s : result) {
logger.info(s);
}
//Expected values and order
assertEquals(result.size(),20);
assertEquals(result.get(0),"-m");
assertEquals(result.get(1),"yarn-cluster");
assertEquals(result.get(2),"-ys");
assertSame(Integer.valueOf(result.get(3)),slot);
assertEquals(result.get(4),"-ynm");
assertEquals(result.get(5),appName);
assertEquals(result.get(6),"-yn");
assertSame(Integer.valueOf(result.get(7)),taskManager);
assertEquals(result.get(8),"-yjm");
assertEquals(result.get(9),jobManagerMemory);
assertEquals(result.get(10),"-ytm");
assertEquals(result.get(11),taskManagerMemory);
assertEquals(result.get(12),"-d");
assertEquals(result.get(13),"-c");
assertEquals(result.get(14),mainClass);
assertEquals(result.get(15),mainJar.getRes());
assertEquals(result.get(16),mainArgs);
assertEquals(result.get(17),"--qu");
assertEquals(result.get(18),queue);
assertEquals(result.get(19),others);
//Others param without --qu
FlinkParameters param1 = new FlinkParameters();
param1.setQueue(queue);
param1.setDeployMode(mode);
result = FlinkArgsUtils.buildArgs(param1);
assertEquals(result.size(),5);
}
}

2
pom.xml

@ -655,6 +655,8 @@
<include>**/alert/utils/JSONUtilsTest.java</include> <include>**/alert/utils/JSONUtilsTest.java</include>
<include>**/alert/utils/PropertyUtilsTest.java</include> <include>**/alert/utils/PropertyUtilsTest.java</include>
<include>**/server/utils/SparkArgsUtilsTest.java</include> <include>**/server/utils/SparkArgsUtilsTest.java</include>
<include>**/server/utils/FlinkArgsUtilsTest.java</include>
<include>**/dao/mapper/AccessTokenMapperTest.java</include>
</includes> </includes>
<!-- <skip>true</skip> --> <!-- <skip>true</skip> -->
</configuration> </configuration>

Loading…
Cancel
Save