github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[fix] [MLflow Task plugin] Sometimes errors are reported using preset algorithms (#10986) (#10987) 

* Optimize the download method of preset project

* fix checkstyle
3.1.0-release
JieguangZhou 2 years ago committed by GitHub
parent
fd5fb85858
commit
11a354927d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 109 additions and 110 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 64
      dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java
  2. 29
      dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java
  3. 122
      dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java

64
dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java
Unescape View File

@ -30,11 +30,13 @@ public class MlflowConstants {
public static final String PRESET_REPOSITORY = "https://github.com/apache/dolphinscheduler-mlflow"; public static final String PRESET_REPOSITORY = "https://github.com/apache/dolphinscheduler-mlflow";
public static final String PRESET_PATH = "dolphinscheduler-mlflow";
public static final String PRESET_REPOSITORY_VERSION = "main"; public static final String PRESET_REPOSITORY_VERSION = "main";
public static final String PRESET_AUTOML_PROJECT = PRESET_REPOSITORY + "#Project-AutoML"; public static final String PRESET_AUTOML_PROJECT = PRESET_PATH + "#Project-AutoML";
public static final String PRESET_BASIC_ALGORITHM_PROJECT = PRESET_REPOSITORY + "#Project-BasicAlgorithm"; public static final String PRESET_BASIC_ALGORITHM_PROJECT = PRESET_PATH + "#Project-BasicAlgorithm";
public static final String MLFLOW_TASK_TYPE_PROJECTS = "MLflow Projects"; public static final String MLFLOW_TASK_TYPE_PROJECTS = "MLflow Projects";
@ -62,27 +64,25 @@ public class MlflowConstants {
public static final String SET_REPOSITORY = "repo=%s"; public static final String SET_REPOSITORY = "repo=%s";
public static final String MLFLOW_RUN_BASIC_ALGORITHM = "mlflow run $repo " + public static final String MLFLOW_RUN_BASIC_ALGORITHM = "mlflow run $repo "
"-P algorithm=%s " + + "-P algorithm=%s "
"-P data_path=$data_path " + + "-P data_path=$data_path "
"-P params=\"%s\" " + + "-P params=\"%s\" "
"-P search_params=\"%s\" " + + "-P search_params=\"%s\" "
"-P model_name=\"%s\" " + + "-P model_name=\"%s\" "
"--experiment-name=\"%s\" " + + "--experiment-name=\"%s\"";
"--version=main ";
public static final String MLFLOW_RUN_AUTOML_PROJECT = "mlflow run $repo "
public static final String MLFLOW_RUN_AUTOML_PROJECT = "mlflow run $repo " + + "-P tool=%s "
"-P tool=%s " + + "-P data_path=$data_path "
"-P data_path=$data_path " + + "-P params=\"%s\" "
"-P params=\"%s\" " + + "-P model_name=\"%s\" "
"-P model_name=\"%s\" " + + "--experiment-name=\"%s\"";
"--experiment-name=\"%s\" " +
"--version=main "; public static final String MLFLOW_RUN_CUSTOM_PROJECT = "mlflow run $repo "
+ "%s "
public static final String MLFLOW_RUN_CUSTOM_PROJECT = "mlflow run $repo " + + "--experiment-name=\"%s\" "
"%s " + + "--version=\"%s\" ";
"--experiment-name=\"%s\" " +
"--version=\"%s\" ";
public static final String MLFLOW_MODELS_SERVE = "mlflow models serve -m %s --port %s -h 0.0.0.0"; public static final String MLFLOW_MODELS_SERVE = "mlflow models serve -m %s --port %s -h 0.0.0.0";
@ -90,17 +90,17 @@ public class MlflowConstants {
public static final String DOCKER_RREMOVE_CONTAINER = "docker rm -f %s"; public static final String DOCKER_RREMOVE_CONTAINER = "docker rm -f %s";
public static final String DOCKER_RUN = "docker run -d --name=%s -p=%s:8080 " + public static final String DOCKER_RUN = "docker run -d --name=%s -p=%s:8080 "
"--health-cmd \"curl --fail http://127.0.0.1:8080/ping || exit 1\" --health-interval 5s --health-retries 20" + + "--health-cmd \"curl --fail http://127.0.0.1:8080/ping || exit 1\" --health-interval 5s --health-retries 20"
" %s"; + " %s";
public static final String DOCKER_COMPOSE_RUN = "docker-compose up -d"; public static final String DOCKER_COMPOSE_RUN = "docker-compose up -d";
public static final String SET_DOCKER_COMPOSE_ENV = "export DS_TASK_MLFLOW_IMAGE_NAME=%s\n" + public static final String SET_DOCKER_COMPOSE_ENV = "export DS_TASK_MLFLOW_IMAGE_NAME=%s\n"
"export DS_TASK_MLFLOW_CONTAINER_NAME=%s\n" + + "export DS_TASK_MLFLOW_CONTAINER_NAME=%s\n"
"export DS_TASK_MLFLOW_DEPLOY_PORT=%s\n" + + "export DS_TASK_MLFLOW_DEPLOY_PORT=%s\n"
"export DS_TASK_MLFLOW_CPU_LIMIT=%s\n" + + "export DS_TASK_MLFLOW_CPU_LIMIT=%s\n"
"export DS_TASK_MLFLOW_MEMORY_LIMIT=%s"; + "export DS_TASK_MLFLOW_MEMORY_LIMIT=%s";
public static final String DOCKER_HEALTH_CHECK = "docker inspect --format \"{{json .State.Health.Status }}\" %s"; public static final String DOCKER_HEALTH_CHECK = "docker inspect --format \"{{json .State.Health.Status }}\" %s";
@ -108,4 +108,6 @@ public class MlflowConstants {
public static final int DOCKER_HEALTH_CHECK_TIMEOUT = 20; public static final int DOCKER_HEALTH_CHECK_TIMEOUT = 20;
public static final int DOCKER_HEALTH_CHECK_INTERVAL = 5000; public static final int DOCKER_HEALTH_CHECK_INTERVAL = 5000;
public static final String GIT_CLONE_REPO = "git clone %s %s";
} }

29
dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java
Unescape View File

@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.task.mlflow;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor; import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor;
import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
@ -26,14 +27,11 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.OSUtils;
import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.OSUtils;
import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -50,12 +48,12 @@ public class MlflowTask extends AbstractTaskExecutor {
/** /**
* shell command executor * shell command executor
*/ */
private ShellCommandExecutor shellCommandExecutor; private final ShellCommandExecutor shellCommandExecutor;
/** /**
* taskExecutionContext * taskExecutionContext
*/ */
private TaskExecutionContext taskExecutionContext; private final TaskExecutionContext taskExecutionContext;
/** /**
* constructor * constructor
@ -87,9 +85,9 @@ public class MlflowTask extends AbstractTaskExecutor {
String command = buildCommand(); String command = buildCommand();
TaskResponse commandExecuteResult = shellCommandExecutor.run(command); TaskResponse commandExecuteResult = shellCommandExecutor.run(command);
int exitCode; int exitCode;
if (mlflowParameters.getIsDeployDocker()){ if (mlflowParameters.getIsDeployDocker()) {
exitCode = checkDockerHealth(); exitCode = checkDockerHealth();
}else { } else {
exitCode = commandExecuteResult.getExitStatusCode(); exitCode = commandExecuteResult.getExitStatusCode();
} }
setExitStatusCode(exitCode); setExitStatusCode(exitCode);
@ -136,19 +134,20 @@ public class MlflowTask extends AbstractTaskExecutor {
if (mlflowParameters.getMlflowJobType().equals(MlflowConstants.JOB_TYPE_BASIC_ALGORITHM)) { if (mlflowParameters.getMlflowJobType().equals(MlflowConstants.JOB_TYPE_BASIC_ALGORITHM)) {
args.add(String.format(MlflowConstants.SET_DATA_PATH, mlflowParameters.getDataPath())); args.add(String.format(MlflowConstants.SET_DATA_PATH, mlflowParameters.getDataPath()));
args.add(String.format(MlflowConstants.SET_REPOSITORY, MlflowConstants.PRESET_BASIC_ALGORITHM_PROJECT)); args.add(String.format(MlflowConstants.SET_REPOSITORY, MlflowConstants.PRESET_BASIC_ALGORITHM_PROJECT));
args.add(String.format(MlflowConstants.GIT_CLONE_REPO, MlflowConstants.PRESET_REPOSITORY, MlflowConstants.PRESET_PATH));
runCommand = MlflowConstants.MLFLOW_RUN_BASIC_ALGORITHM; runCommand = MlflowConstants.MLFLOW_RUN_BASIC_ALGORITHM;
runCommand = String.format(runCommand, mlflowParameters.getAlgorithm(), mlflowParameters.getParams(), mlflowParameters.getSearchParams(), mlflowParameters.getModelName(), mlflowParameters.getExperimentName()); runCommand = String.format(runCommand, mlflowParameters.getAlgorithm(), mlflowParameters.getParams(), mlflowParameters.getSearchParams(), mlflowParameters.getModelName(),
mlflowParameters.getExperimentName());
} else if (mlflowParameters.getMlflowJobType().equals(MlflowConstants.JOB_TYPE_AUTOML)) { } else if (mlflowParameters.getMlflowJobType().equals(MlflowConstants.JOB_TYPE_AUTOML)) {
args.add(String.format(MlflowConstants.SET_DATA_PATH, mlflowParameters.getDataPath())); args.add(String.format(MlflowConstants.SET_DATA_PATH, mlflowParameters.getDataPath()));
args.add(String.format(MlflowConstants.SET_REPOSITORY, MlflowConstants.PRESET_AUTOML_PROJECT)); args.add(String.format(MlflowConstants.SET_REPOSITORY, MlflowConstants.PRESET_AUTOML_PROJECT));
args.add(String.format(MlflowConstants.GIT_CLONE_REPO, MlflowConstants.PRESET_REPOSITORY, MlflowConstants.PRESET_PATH));
runCommand = MlflowConstants.MLFLOW_RUN_AUTOML_PROJECT; runCommand = MlflowConstants.MLFLOW_RUN_AUTOML_PROJECT;
runCommand = String.format(runCommand, mlflowParameters.getAutomlTool(), mlflowParameters.getParams(), mlflowParameters.getModelName(), mlflowParameters.getExperimentName()); runCommand = String.format(runCommand, mlflowParameters.getAutomlTool(), mlflowParameters.getParams(), mlflowParameters.getModelName(), mlflowParameters.getExperimentName());
} else if (mlflowParameters.getMlflowJobType().equals(MlflowConstants.JOB_TYPE_CUSTOM_PROJECT)) { } else if (mlflowParameters.getMlflowJobType().equals(MlflowConstants.JOB_TYPE_CUSTOM_PROJECT)) {
args.add(String.format(MlflowConstants.SET_REPOSITORY, mlflowParameters.getMlflowProjectRepository())); args.add(String.format(MlflowConstants.SET_REPOSITORY, mlflowParameters.getMlflowProjectRepository()));
@ -166,10 +165,9 @@ public class MlflowTask extends AbstractTaskExecutor {
protected String buildCommandForMlflowModels() { protected String buildCommandForMlflowModels() {
/** /**
* papermill [OPTIONS] NOTEBOOK_PATH [OUTPUT_PATH] * build mlflow models command
*/ */
Map<String, Property> paramsMap = getParamsMap(); Map<String, Property> paramsMap = getParamsMap();
List<String> args = new ArrayList<>(); List<String> args = new ArrayList<>();
args.add(String.format(MlflowConstants.EXPORT_MLFLOW_TRACKING_URI_ENV, mlflowParameters.getMlflowTrackingUri())); args.add(String.format(MlflowConstants.EXPORT_MLFLOW_TRACKING_URI_ENV, mlflowParameters.getMlflowTrackingUri()));
@ -211,7 +209,7 @@ public class MlflowTask extends AbstractTaskExecutor {
logger.info("checking container healthy ... "); logger.info("checking container healthy ... ");
int exitCode = -1; int exitCode = -1;
String[] command = {"sh", "-c", String.format(MlflowConstants.DOCKER_HEALTH_CHECK, mlflowParameters.getContainerName())}; String[] command = {"sh", "-c", String.format(MlflowConstants.DOCKER_HEALTH_CHECK, mlflowParameters.getContainerName())};
for(int x = 0; x < MlflowConstants.DOCKER_HEALTH_CHECK_TIMEOUT; x = x+1) { for (int x = 0; x < MlflowConstants.DOCKER_HEALTH_CHECK_TIMEOUT; x = x + 1) {
String status; String status;
try { try {
status = OSUtils.exeShell(command).replace("\n", "").replace("\"", ""); status = OSUtils.exeShell(command).replace("\n", "").replace("\"", "");
@ -224,7 +222,7 @@ public class MlflowTask extends AbstractTaskExecutor {
exitCode = 0; exitCode = 0;
logger.info("container is healthy"); logger.info("container is healthy");
return exitCode; return exitCode;
}else { } else {
logger.info("The health check has been running for {} seconds", x * MlflowConstants.DOCKER_HEALTH_CHECK_INTERVAL / 1000); logger.info("The health check has been running for {} seconds", x * MlflowConstants.DOCKER_HEALTH_CHECK_INTERVAL / 1000);
ThreadUtils.sleep(MlflowConstants.DOCKER_HEALTH_CHECK_INTERVAL); ThreadUtils.sleep(MlflowConstants.DOCKER_HEALTH_CHECK_INTERVAL);
} }
@ -234,7 +232,6 @@ public class MlflowTask extends AbstractTaskExecutor {
return exitCode; return exitCode;
} }
@Override @Override
public AbstractParameters getParameters() { public AbstractParameters getParameters() {
return mlflowParameters; return mlflowParameters;

122
dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java
Unescape View File

@ -17,30 +17,29 @@
package org.apache.dolphinler.plugin.task.mlflow; package org.apache.dolphinler.plugin.task.mlflow;
import java.util.Date;
import java.util.UUID;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager;
import org.apache.dolphinscheduler.plugin.task.api.utils.OSUtils;
import org.apache.dolphinscheduler.plugin.task.mlflow.MlflowConstants; import org.apache.dolphinscheduler.plugin.task.mlflow.MlflowConstants;
import org.apache.dolphinscheduler.plugin.task.mlflow.MlflowParameters; import org.apache.dolphinscheduler.plugin.task.mlflow.MlflowParameters;
import org.apache.dolphinscheduler.plugin.task.mlflow.MlflowTask; import org.apache.dolphinscheduler.plugin.task.mlflow.MlflowTask;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import java.util.Date;
import java.util.UUID;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito; import org.powermock.api.mockito.PowerMockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.junit.runner.RunWith; import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@RunWith(PowerMockRunner.class) @RunWith(PowerMockRunner.class)
@PrepareForTest({ @PrepareForTest({
@ -81,84 +80,85 @@ public class MlflowTaskTest {
public void testInitBasicAlgorithmTask() { public void testInitBasicAlgorithmTask() {
MlflowTask mlflowTask = initTask(createBasicAlgorithmParameters()); MlflowTask mlflowTask = initTask(createBasicAlgorithmParameters());
Assert.assertEquals(mlflowTask.buildCommand(), Assert.assertEquals(mlflowTask.buildCommand(),
"export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" + "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n"
"data_path=/data/iris.csv\n" + + "data_path=/data/iris.csv\n"
"repo=https://github.com/apache/dolphinscheduler-mlflow#Project-BasicAlgorithm\n" + + "repo=dolphinscheduler-mlflow#Project-BasicAlgorithm\n"
"mlflow run $repo " + + "git clone https://github.com/apache/dolphinscheduler-mlflow dolphinscheduler-mlflow\n"
"-P algorithm=xgboost " + + "mlflow run $repo "
"-P data_path=$data_path " + + "-P algorithm=xgboost "
"-P params=\"n_estimators=100\" " + + "-P data_path=$data_path "
"-P search_params=\"\" " + + "-P params=\"n_estimators=100\" "
"-P model_name=\"BasicAlgorithm\" " + + "-P search_params=\"\" "
"--experiment-name=\"BasicAlgorithm\" " + + "-P model_name=\"BasicAlgorithm\" "
"--version=main "); + "--experiment-name=\"BasicAlgorithm\"");
} }
@Test @Test
public void testInitAutoMLTask() { public void testInitAutoMLTask() {
MlflowTask mlflowTask = initTask(createAutoMLParameters()); MlflowTask mlflowTask = initTask(createAutoMLParameters());
Assert.assertEquals(mlflowTask.buildCommand(), Assert.assertEquals(mlflowTask.buildCommand(),
"export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" + "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n"
"data_path=/data/iris.csv\n" + + "data_path=/data/iris.csv\n"
"repo=https://github.com/apache/dolphinscheduler-mlflow#Project-AutoML\n" + + "repo=dolphinscheduler-mlflow#Project-AutoML\n"
"mlflow run $repo " + + "git clone https://github.com/apache/dolphinscheduler-mlflow dolphinscheduler-mlflow\n"
"-P tool=autosklearn " + + "mlflow run $repo "
"-P data_path=$data_path " + + "-P tool=autosklearn "
"-P params=\"time_left_for_this_task=30\" " + + "-P data_path=$data_path "
"-P model_name=\"AutoML\" " + + "-P params=\"time_left_for_this_task=30\" "
"--experiment-name=\"AutoML\" " + + "-P model_name=\"AutoML\" "
"--version=main "); + "--experiment-name=\"AutoML\"");
} }
@Test @Test
public void testInitCustomProjectTask() { public void testInitCustomProjectTask() {
MlflowTask mlflowTask = initTask(createCustomProjectParameters()); MlflowTask mlflowTask = initTask(createCustomProjectParameters());
Assert.assertEquals(mlflowTask.buildCommand(), Assert.assertEquals(mlflowTask.buildCommand(),
"export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" + "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n"
"repo=https://github.com/mlflow/mlflow#examples/xgboost/xgboost_native\n" + + "repo=https://github.com/mlflow/mlflow#examples/xgboost/xgboost_native\n"
"mlflow run $repo " + + "mlflow run $repo "
"-P learning_rate=0.2 " + + "-P learning_rate=0.2 "
"-P colsample_bytree=0.8 " + + "-P colsample_bytree=0.8 "
"-P subsample=0.9 " + + "-P subsample=0.9 "
"--experiment-name=\"custom_project\" " + + "--experiment-name=\"custom_project\" "
"--version=\"master\" "); + "--version=\"master\" ");
} }
@Test @Test
public void testModelsDeployMlflow() { public void testModelsDeployMlflow() {
MlflowTask mlflowTask = initTask(createModelDeplyMlflowParameters()); MlflowTask mlflowTask = initTask(createModelDeplyMlflowParameters());
Assert.assertEquals(mlflowTask.buildCommand(), Assert.assertEquals(mlflowTask.buildCommand(),
"export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" + "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n"
"mlflow models serve -m models:/model/1 --port 7000 -h 0.0.0.0"); + "mlflow models serve -m models:/model/1 --port 7000 -h 0.0.0.0");
} }
@Test @Test
public void testModelsDeployDocker() { public void testModelsDeployDocker() {
MlflowTask mlflowTask = initTask(createModelDeplyDockerParameters()); MlflowTask mlflowTask = initTask(createModelDeplyDockerParameters());
Assert.assertEquals(mlflowTask.buildCommand(), Assert.assertEquals(mlflowTask.buildCommand(),
"export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" + "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n"
"mlflow models build-docker -m models:/model/1 -n mlflow/model:1 --enable-mlserver\n" + + "mlflow models build-docker -m models:/model/1 -n mlflow/model:1 --enable-mlserver\n"
"docker rm -f ds-mlflow-model-1\n" + + "docker rm -f ds-mlflow-model-1\n"
"docker run -d --name=ds-mlflow-model-1 -p=7000:8080 " + + "docker run -d --name=ds-mlflow-model-1 -p=7000:8080 "
"--health-cmd \"curl --fail http://127.0.0.1:8080/ping || exit 1\" --health-interval 5s --health-retries 20 " + + "--health-cmd \"curl --fail http://127.0.0.1:8080/ping || exit 1\" --health-interval 5s --health-retries 20 "
"mlflow/model:1"); + "mlflow/model:1");
} }
@Test @Test
public void testModelsDeployDockerCompose() throws Exception{ public void testModelsDeployDockerCompose() throws Exception {
MlflowTask mlflowTask = initTask(createModelDeplyDockerComposeParameters()); MlflowTask mlflowTask = initTask(createModelDeplyDockerComposeParameters());
Assert.assertEquals(mlflowTask.buildCommand(), Assert.assertEquals(mlflowTask.buildCommand(),
"export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" + "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n"
"cp " + mlflowTask.getTemplatePath(MlflowConstants.TEMPLATE_DOCKER_COMPOSE) + + "cp "
" /tmp/dolphinscheduler_test\n" + + mlflowTask.getTemplatePath(MlflowConstants.TEMPLATE_DOCKER_COMPOSE)
"mlflow models build-docker -m models:/model/1 -n mlflow/model:1 --enable-mlserver\n" + + " /tmp/dolphinscheduler_test\n"
"docker rm -f ds-mlflow-model-1\n" + + "mlflow models build-docker -m models:/model/1 -n mlflow/model:1 --enable-mlserver\n"
"export DS_TASK_MLFLOW_IMAGE_NAME=mlflow/model:1\n" + + "docker rm -f ds-mlflow-model-1\n"
"export DS_TASK_MLFLOW_CONTAINER_NAME=ds-mlflow-model-1\n" + + "export DS_TASK_MLFLOW_IMAGE_NAME=mlflow/model:1\n"
"export DS_TASK_MLFLOW_DEPLOY_PORT=7000\n" + + "export DS_TASK_MLFLOW_CONTAINER_NAME=ds-mlflow-model-1\n"
"export DS_TASK_MLFLOW_CPU_LIMIT=0.5\n" + + "export DS_TASK_MLFLOW_DEPLOY_PORT=7000\n"
"export DS_TASK_MLFLOW_MEMORY_LIMIT=200m\n" + + "export DS_TASK_MLFLOW_CPU_LIMIT=0.5\n"
"docker-compose up -d"); + "export DS_TASK_MLFLOW_MEMORY_LIMIT=200m\n"
+ "docker-compose up -d");
} }
private MlflowTask initTask(MlflowParameters mlflowParameters) { private MlflowTask initTask(MlflowParameters mlflowParameters) {

Write Preview
Loading…
Cancel
Save