DolphinScheduler/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/spark/SparkTaskTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.dolphinscheduler.server.worker.task.spark;

import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.PropertyPlaceholderHelper;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;

import java.util.Date;

import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;

@RunWith(PowerMockRunner.class)
@PrepareForTest({ParameterUtils.class, PlaceholderUtils.class, PropertyPlaceholderHelper.class})
public class SparkTaskTest {

    private static final Logger logger = LoggerFactory.getLogger(SparkTaskTest.class);

    private TaskExecutionContext taskExecutionContext;

    private ApplicationContext applicationContext;

    private ProcessService processService;

    private SparkTask spark2Task;

    String spark1Params = "{"
        + "\"mainArgs\":\"\", "
        + "\"driverMemory\":\"1G\", "
        + "\"executorMemory\":\"2G\", "
        + "\"programType\":\"SCALA\", "
        + "\"mainClass\":\"basicetl.GlobalUserCar\", "
        + "\"driverCores\":\"2\", "
        + "\"deployMode\":\"cluster\", "
        + "\"executorCores\":2, "
        + "\"mainJar\":{\"res\":\"test-1.0-SNAPSHOT.jar\"}, "
        + "\"sparkVersion\":\"SPARK1\", "
        + "\"numExecutors\":\"10\", "
        + "\"localParams\":[], "
        + "\"others\":\"\", "
        + "\"resourceList\":[]"
        + "}";

    String spark2Params = "{"
        + "\"mainArgs\":\"\", "
        + "\"driverMemory\":\"1G\", "
        + "\"executorMemory\":\"2G\", "
        + "\"programType\":\"SCALA\", "
        + "\"mainClass\":\"basicetl.GlobalUserCar\", "
        + "\"driverCores\":\"2\", "
        + "\"deployMode\":\"cluster\", "
        + "\"executorCores\":2, "
        + "\"mainJar\":{\"res\":\"test-1.0-SNAPSHOT.jar\"}, "
        + "\"sparkVersion\":\"SPARK2\", "
        + "\"numExecutors\":\"10\", "
        + "\"localParams\":[], "
        + "\"others\":\"\", "
        + "\"resourceList\":[]"
        + "}";

    @Before
    public void setTaskExecutionContext() {
        taskExecutionContext = new TaskExecutionContext();
        taskExecutionContext.setTaskParams(spark2Params);
        taskExecutionContext.setQueue("dev");
        taskExecutionContext.setTaskAppId(String.valueOf(System.currentTimeMillis()));
        taskExecutionContext.setTenantCode("1");
        taskExecutionContext.setEnvFile(".dolphinscheduler_env.sh");
        taskExecutionContext.setStartTime(new Date());
        taskExecutionContext.setTaskTimeout(0);

        processService = Mockito.mock(ProcessService.class);
        applicationContext = Mockito.mock(ApplicationContext.class);
        SpringApplicationContext springApplicationContext = new SpringApplicationContext();
        springApplicationContext.setApplicationContext(applicationContext);
        Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);

        spark2Task = new SparkTask(taskExecutionContext, logger);
        spark2Task.init();
    }

    @Test
    public void testSparkTaskInit() {

        TaskExecutionContext sparkTaskCtx = new TaskExecutionContext();
        SparkTask sparkTask = new SparkTask(sparkTaskCtx, logger);
        sparkTask.init();
        sparkTask.getParameters();
        Assert.assertNull(sparkTaskCtx.getTaskParams());

        String spark2Command = spark2Task.buildCommand();
        String spark2Expected = "${SPARK_HOME2}/bin/spark-submit --master yarn --deploy-mode cluster "
            + "--class basicetl.GlobalUserCar --driver-cores 2 --driver-memory 1G --num-executors 10 "
            + "--executor-cores 2 --executor-memory 2G --queue dev test-1.0-SNAPSHOT.jar";
        Assert.assertEquals(spark2Expected, spark2Command);

        taskExecutionContext.setTaskParams(spark1Params);

        SparkTask spark1Task = new SparkTask(taskExecutionContext, logger);
        spark1Task.init();
        String spark1Command = spark1Task.buildCommand();
        String spark1Expected = "${SPARK_HOME1}/bin/spark-submit --master yarn --deploy-mode cluster "
            + "--class basicetl.GlobalUserCar --driver-cores 2 --driver-memory 1G --num-executors 10 "
            + "--executor-cores 2 --executor-memory 2G --queue dev test-1.0-SNAPSHOT.jar";
        Assert.assertEquals(spark1Expected, spark1Command);
    }
}