diff --git a/docs/docs/en/guide/task/mlflow.md b/docs/docs/en/guide/task/mlflow.md index 486bf601e8..8e15dd753e 100644 --- a/docs/docs/en/guide/task/mlflow.md +++ b/docs/docs/en/guide/task/mlflow.md @@ -13,10 +13,10 @@ MLflow task plugin used to execute MLflow tasks，Currently contains Mlflow Proj The Mlflow plugin currently supports and will support the following: -- [ ] MLflow Projects - - [x] BasicAlgorithm: contains lr, svm, lightgbm, xgboost +- [x] MLflow Projects + - [x] BasicAlgorithm: contains LogisticRegression, svm, lightgbm, xgboost - [x] AutoML: AutoML tool，contains autosklean, flaml - - [ ] Custom projects: Support for running your own MLflow projects + - [x] Custom projects: Support for running your own MLflow projects - [ ] MLflow Models - [x] MLFLOW: Use `MLflow models serve` to deploy a model service - [x] Docker: Run the container after packaging the docker image @@ -65,9 +65,7 @@ First, introduce some general parameters of DolphinScheduler **Task Parameter** - **mlflow server tracking uri** ：MLflow server uri, default http://localhost:5000. -- **job type** : The type of task to run, currently including the underlying algorithm and AutoML. (User-defined - MLFlow project task execution will be supported in the near future) -- **experiment name** ：The experiment in which the task is running, if none, is created. +- **experiment name** ：Create the experiment where the task is running, if the experiment does not exist. If the name is empty, it is set to ` Default `, the same as MLflow. - **register model** ：Register the model or not. If register is selected, the following parameters are expanded. - **model name** : The registered model name is added to the original model version and registered as Production. @@ -95,9 +93,7 @@ First, introduce some general parameters of DolphinScheduler **Task Parameter** - **mlflow server tracking uri** ：MLflow server uri, default http://localhost:5000. -- **job type** : The type of task to run, currently including the underlying algorithm and AutoML. (User-defined - MLFlow project task execution will be supported in the near future) -- **experiment name** ：The experiment in which the task is running, if none, is created. +- **experiment name** ：Create the experiment where the task is running, if the experiment does not exist. If the name is empty, it is set to ` Default `, the same as MLflow. - **register model** ：Register the model or not. If register is selected, the following parameters are expanded. - **model name** : The registered model name is added to the original model version and registered as Production. @@ -114,6 +110,24 @@ First, introduce some general parameters of DolphinScheduler and [flaml](https://github.com/microsoft/FLAML) +#### Custom projects + +![mlflow-custom-project-template.png](/img/tasks/demo/mlflow-custom-project-template.png) + +**Task Parameter** + +- **mlflow server tracking uri** ：MLflow server uri, default http://localhost:5000. +- **experiment name** ：Create the experiment where the task is running, if the experiment does not exist. If the name is empty, it is set to ` Default `, the same as MLflow. +- **parameters** : `--param-list` in `mlflow run`. For example `-P learning_rate=0.2 -P colsample_bytree=0.8 -P subsample=0.9` +- **Repository** : Repository url of MLflow Project，Support git address and directory on worker. If it's in a subdirectory，We add `#` to support this (same as `mlflow run`) , for example `https://github.com/mlflow/mlflow#examples/xgboost/xgboost_native` +- **Project Version** : Version of the project，default master + +You can now use this feature to run all mlFlow projects on Github (For example [MLflow examples](https://github.com/mlflow/mlflow/tree/master/examples) )了。You can also create your own machine learning library to reuse your work, and then use DolphinScheduler to use your library with one click. + +The actual interface is as follows + +![mlflow-custom-project.png](/img/tasks/demo/mlflow-custom-project.png) + ### MLflow Models #### MLFLOW diff --git a/docs/docs/zh/guide/task/mlflow.md b/docs/docs/zh/guide/task/mlflow.md index 507ef43ce6..52fd4d32ef 100644 --- a/docs/docs/zh/guide/task/mlflow.md +++ b/docs/docs/zh/guide/task/mlflow.md @@ -12,10 +12,10 @@ MLflow 组件用于执行 MLflow 任务，目前包含Mlflow Projects, 和MLflow 目前 Mlflow 组件支持的和即将支持的内容如下中： -- [ ] MLflow Projects - - [x] BasicAlgorithm: 基础算法，包含lr, svm, lightgbm, xgboost。 +- [x] MLflow Projects + - [x] BasicAlgorithm: 基础算法，包含LogisticRegression, svm, lightgbm, xgboost。 - [x] AutoML: AutoML工具，包含autosklean, flaml。 - - [ ] Custom projects: 支持运行自己的MLflow Projects项目 + - [x] Custom projects: 支持运行自己的MLflow Projects项目 - [ ] MLflow Models - [x] MLFLOW: 直接使用 `MLflow models serve` 部署模型 - [x] Docker: 打包 DOCKER 镜像后部署模型 @@ -57,13 +57,12 @@ MLflow 组件用于执行 MLflow 任务，目前包含Mlflow Projects, 和MLflow **任务参数** - **mlflow server tracking uri** ：MLflow server 的连接, 默认 http://localhost:5000。 -- **任务类型** : 运行的任务类型，目前包括基础算法与AutoML, 后续将会支持用户自定义的MLflow Project。 -- **实验名称** ：任务运行时所在的实验，若无则创建。 +- **实验名称** ：任务运行时所在的实验，若实验不存在，则创建。若实验名称为空，则设置为`Default`, 与 MLflow 一样。 - **注册模型** ：是否注册模型，若选择注册，则会展开以下参数。 - **注册的模型名称** : 注册的模型名称，会在原来的基础上加上一个模型版本，并注册为Production。 - **数据路径** : 文件/文件夹的绝对路径, 若文件需以.csv结尾（自动切分训练集与测试集）, 文件夹需包含train.csv和test.csv（建议方式，用户应自行构建测试集用于模型评估）。 详细的参数列表如下: - - [lr](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html#sklearn.linear_model.LogisticRegression) + - [LogisticRegression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html#sklearn.linear_model.LogisticRegression) - [SVM](https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html?highlight=svc#sklearn.svm.SVC) - [lightgbm](https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMClassifier.html#lightgbm.LGBMClassifier) - [xgboost](https://xgboost.readthedocs.io/en/stable/python/python_api.html#xgboost.XGBClassifier) @@ -77,8 +76,7 @@ MLflow 组件用于执行 MLflow 任务，目前包含Mlflow Projects, 和MLflow **任务参数** - **mlflow server tracking uri** ：MLflow server 的连接, 默认 http://localhost:5000。 -- **任务类型** : 运行的任务类型，目前包括基础算法与AutoML, 后续将会支持用户自定义的MLflow Project。 -- **实验名称** ：任务运行时所在的实验，若无则创建。 +- **实验名称** ：任务运行时所在的实验，若实验不存在，则创建。若实验名称为空，则设置为`Default`, 与 MLflow 一样。 - **注册模型** ：是否注册模型，若选择注册，则会展开以下参数。 - **注册的模型名称** : 注册的模型名称，会在原来的基础上加上一个模型版本，并注册为Production。 - **数据路径** : 文件/文件夹的绝对路径, 若文件需以.csv结尾（自动切分训练集与测试集）, 文件夹需包含train.csv和test.csv（建议方式，用户应自行构建测试集用于模型评估）。 @@ -88,6 +86,24 @@ MLflow 组件用于执行 MLflow 任务，目前包含Mlflow Projects, 和MLflow - **AutoML工具** : 使用的AutoML工具，目前支持 [autosklearn](https://github.com/automl/auto-sklearn) , [flaml](https://github.com/microsoft/FLAML) +#### Custom projects + +![mlflow-custom-project-template.png](/img/tasks/demo/mlflow-custom-project-template.png) + +**任务参数** + +- **mlflow server tracking uri** ：MLflow server 的连接, 默认 http://localhost:5000。 +- **实验名称** ：任务运行时所在的实验，若实验不存在，则创建。若实验名称为空，则设置为`Default`, 与 MLflow 一样。 +- **参数** : `mlflow run`中的 --param-list 如 `-P learning_rate=0.2 -P colsample_bytree=0.8 -P subsample=0.9` +- **运行仓库** : MLflow Project的仓库地址，可以为github地址，或者worker上的目录, 如Mlflow project位于子目录，可以添加 `#` 隔开, 如 `https://github.com/mlflow/mlflow#examples/xgboost/xgboost_native` +- **项目版本** : 对应项目中git版本管理中的版本，默认 master + +现在你可以使用这个功能来运行github上所有的MLflow Projects (如 [MLflow examples](https://github.com/mlflow/mlflow/tree/master/examples) )了。你也可以创建自己的机器学习库，用来复用你的研究成果，以后你就可以使用DolphinScheduler来一键操作使用你的算法库。 + +实际运行界面如下 + +![mlflow-custom-project.png](/img/tasks/demo/mlflow-custom-project.png) + ### MLflow Models diff --git a/docs/img/tasks/demo/mlflow-custom-project-template.png b/docs/img/tasks/demo/mlflow-custom-project-template.png new file mode 100644 index 0000000000..cbb2d6f732 Binary files /dev/null and b/docs/img/tasks/demo/mlflow-custom-project-template.png differ diff --git a/docs/img/tasks/demo/mlflow-custom-project.png b/docs/img/tasks/demo/mlflow-custom-project.png new file mode 100644 index 0000000000..54a8cb0ab8 Binary files /dev/null and b/docs/img/tasks/demo/mlflow-custom-project.png differ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java index f318f50966..92d849e433 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java @@ -26,6 +26,8 @@ public class MlflowConstants { public static final String JOB_TYPE_BASIC_ALGORITHM = "BasicAlgorithm"; + public static final String JOB_TYPE_CUSTOM_PROJECT = "CustomProject"; + public static final String PRESET_REPOSITORY = "https://github.com/apache/dolphinscheduler-mlflow"; public static final String PRESET_REPOSITORY_VERSION = "main"; @@ -75,9 +77,14 @@ public class MlflowConstants { "--experiment-name=\"%s\" " + "--version=main "; + public static final String MLFLOW_RUN_CUSTOM_PROJECT = "mlflow run $repo " + + "%s " + + "--experiment-name=\"%s\" " + + "--version=\"%s\" "; + public static final String MLFLOW_MODELS_SERVE = "mlflow models serve -m %s --port %s -h 0.0.0.0"; - public static final String MLFLOW_BUILD_DOCKER = "mlflow models build-docker -m %s -n %s --enable-mlserver"; + public static final String MLFLOW_BUILD_DOCKER = "mlflow models build-docker -m %s -n %s --enable-mlserver"; public static final String DOCKER_RREMOVE_CONTAINER = "docker rm -f %s"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java index 9d107423fd..3557a0698a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java @@ -31,6 +31,13 @@ public class MlflowParameters extends AbstractParameters { private String mlflowJobType = ""; + /** + * CustomProject parameters + */ + private String mlflowProjectRepository; + + private String mlflowProjectVersion = "master"; + /** * AutoML parameters */ @@ -53,7 +60,7 @@ public class MlflowParameters extends AbstractParameters { private String mlflowTaskType = ""; - private String experimentName; + private String experimentName = "Default"; private String modelName = ""; @@ -145,6 +152,22 @@ public class MlflowParameters extends AbstractParameters { this.automlTool = automlTool; } + public String getMlflowProjectRepository() { + return mlflowProjectRepository; + } + + public void setMlflowProjectRepository(String mlflowProjectRepository) { + this.mlflowProjectRepository = mlflowProjectRepository; + } + + public String getMlflowProjectVersion() { + return mlflowProjectVersion; + } + + public void setMlflowProjectVersion(String mlflowProjectVersion) { + this.mlflowProjectVersion = mlflowProjectVersion; + } + public String getAutomlTool() { return automlTool; } @@ -232,7 +255,7 @@ public class MlflowParameters extends AbstractParameters { return scriptPath; } - public String getModelKeyName(String tag) throws IllegalArgumentException{ + public String getModelKeyName(String tag) throws IllegalArgumentException { String imageName; if (deployModelKey.startsWith("runs:")) { imageName = deployModelKey.replace("runs:/", ""); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java index ea87088e6f..226efb21cf 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java @@ -141,8 +141,14 @@ public class MlflowTask extends AbstractTaskExecutor { runCommand = String.format(runCommand, mlflowParameters.getAutomlTool(), mlflowParameters.getParams(), mlflowParameters.getModelName(), mlflowParameters.getExperimentName()); - } else { - runCommand = String.format("Cant not Support %s", mlflowParameters.getMlflowTaskType()); + } else if (mlflowParameters.getMlflowJobType().equals(MlflowConstants.JOB_TYPE_CUSTOM_PROJECT)) { + args.add(String.format(MlflowConstants.SET_REPOSITORY, mlflowParameters.getMlflowProjectRepository())); + + runCommand = MlflowConstants.MLFLOW_RUN_CUSTOM_PROJECT; + runCommand = String.format(runCommand, mlflowParameters.getParams(), mlflowParameters.getExperimentName(), mlflowParameters.getMlflowProjectVersion()); + } + else { + runCommand = String.format("Cant not Support %s", mlflowParameters.getMlflowJobType()); } args.add(runCommand); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java index bf27835d20..24a8dfc601 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java @@ -78,28 +78,73 @@ public class MlflowTaskTest { } @Test - public void testInitBasicAlgorithmTask() throws Exception { + public void testInitBasicAlgorithmTask() { MlflowTask mlflowTask = initTask(createBasicAlgorithmParameters()); - String command = mlflowTask.buildCommand(); - + Assert.assertEquals(mlflowTask.buildCommand(), + "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" + + "data_path=/data/iris.csv\n" + + "repo=https://github.com/apache/dolphinscheduler-mlflow#Project-BasicAlgorithm\n" + + "mlflow run $repo " + + "-P algorithm=xgboost " + + "-P data_path=$data_path " + + "-P params=\"n_estimators=100\" " + + "-P search_params=\"\" " + + "-P model_name=\"BasicAlgorithm\" " + + "--experiment-name=\"BasicAlgorithm\" " + + "--version=main "); } @Test - public void testInitAutoMLTask() throws Exception { + public void testInitAutoMLTask() { MlflowTask mlflowTask = initTask(createAutoMLParameters()); - String command = mlflowTask.buildCommand(); + Assert.assertEquals(mlflowTask.buildCommand(), + "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" + + "data_path=/data/iris.csv\n" + + "repo=https://github.com/apache/dolphinscheduler-mlflow#Project-AutoML\n" + + "mlflow run $repo " + + "-P tool=autosklearn " + + "-P data_path=$data_path " + + "-P params=\"time_left_for_this_task=30\" " + + "-P model_name=\"AutoML\" " + + "--experiment-name=\"AutoML\" " + + "--version=main "); + } + + @Test + public void testInitCustomProjectTask() { + MlflowTask mlflowTask = initTask(createCustomProjectParameters()); + Assert.assertEquals(mlflowTask.buildCommand(), + "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" + + "repo=https://github.com/mlflow/mlflow#examples/xgboost/xgboost_native\n" + + "mlflow run $repo " + + "-P learning_rate=0.2 " + + "-P colsample_bytree=0.8 " + + "-P subsample=0.9 " + + "--experiment-name=\"custom_project\" " + + "--version=\"master\" "); } @Test - public void testModelsDeployMlflow() throws Exception { + public void testModelsDeployMlflow() { MlflowTask mlflowTask = initTask(createModelDeplyMlflowParameters()); - String command = mlflowTask.buildCommand(); + Assert.assertEquals(mlflowTask.buildCommand(), + "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" + + "mlflow models serve -m runs:/a272ec279fc34a8995121ae04281585f/model " + + "--port 7000 " + + "-h 0.0.0.0"); } @Test public void testModelsDeployDocker() throws Exception { MlflowTask mlflowTask = initTask(createModelDeplyDockerParameters()); - String command = mlflowTask.buildCommand(); + Assert.assertEquals(mlflowTask.buildCommand(), + "export MLFLOW_TRACKING_URI=http://127.0.0.1:5000\n" + + "mlflow models build-docker -m runs:/a272ec279fc34a8995121ae04281585f/model " + + "-n mlflow/a272ec279fc34a8995121ae04281585f:model " + + "--enable-mlserver\n" + + "docker rm -f mlflow-a272ec279fc34a8995121ae04281585f-model\n" + + "docker run --name=mlflow-a272ec279fc34a8995121ae04281585f-model " + + "-p=7000:8080 mlflow/a272ec279fc34a8995121ae04281585f:model"); } private MlflowTask initTask(MlflowParameters mlflowParameters) { @@ -116,8 +161,10 @@ public class MlflowTaskTest { mlflowParameters.setMlflowTaskType(MlflowConstants.MLFLOW_TASK_TYPE_PROJECTS); mlflowParameters.setMlflowJobType(MlflowConstants.JOB_TYPE_BASIC_ALGORITHM); mlflowParameters.setAlgorithm("xgboost"); - mlflowParameters.setDataPaths("xxxxxxxxxx"); - mlflowParameters.setExperimentNames("asbbb"); + mlflowParameters.setDataPaths("/data/iris.csv"); + mlflowParameters.setParams("n_estimators=100"); + mlflowParameters.setExperimentNames("BasicAlgorithm"); + mlflowParameters.setModelNames("BasicAlgorithm"); mlflowParameters.setMlflowTrackingUris("http://127.0.0.1:5000"); return mlflowParameters; } @@ -128,13 +175,25 @@ public class MlflowTaskTest { mlflowParameters.setMlflowJobType(MlflowConstants.JOB_TYPE_AUTOML); mlflowParameters.setAutomlTool("autosklearn"); mlflowParameters.setParams("time_left_for_this_task=30"); - mlflowParameters.setDataPaths("xxxxxxxxxxx"); - mlflowParameters.setExperimentNames("asbbb"); - mlflowParameters.setModelNames("asbbb"); + mlflowParameters.setDataPaths("/data/iris.csv"); + mlflowParameters.setExperimentNames("AutoML"); + mlflowParameters.setModelNames("AutoML"); mlflowParameters.setMlflowTrackingUris("http://127.0.0.1:5000"); return mlflowParameters; } + private MlflowParameters createCustomProjectParameters() { + MlflowParameters mlflowParameters = new MlflowParameters(); + mlflowParameters.setMlflowTaskType(MlflowConstants.MLFLOW_TASK_TYPE_PROJECTS); + mlflowParameters.setMlflowJobType(MlflowConstants.JOB_TYPE_CUSTOM_PROJECT); + mlflowParameters.setMlflowTrackingUris("http://127.0.0.1:5000"); + mlflowParameters.setExperimentNames("custom_project"); + mlflowParameters.setParams("-P learning_rate=0.2 -P colsample_bytree=0.8 -P subsample=0.9"); + mlflowParameters.setMlflowProjectRepository("https://github.com/mlflow/mlflow#examples/xgboost/xgboost_native"); + + return mlflowParameters; + } + private MlflowParameters createModelDeplyMlflowParameters() { MlflowParameters mlflowParameters = new MlflowParameters(); mlflowParameters.setMlflowTaskType(MlflowConstants.MLFLOW_TASK_TYPE_MODELS); diff --git a/dolphinscheduler-ui/src/locales/modules/en_US.ts b/dolphinscheduler-ui/src/locales/modules/en_US.ts index bef4da1f40..9e082fdea1 100644 --- a/dolphinscheduler-ui/src/locales/modules/en_US.ts +++ b/dolphinscheduler-ui/src/locales/modules/en_US.ts @@ -993,6 +993,10 @@ const project = { mlflow_deployType: 'Deploy Mode', mlflow_deployModelKey: 'model-uri', mlflow_deployPort: 'Port', + mlflowProjectRepository: 'Repository', + mlflowProjectRepository_tips: 'github respository or path on worker', + mlflowProjectVersion: 'Project Version', + mlflowProjectVersion_tips: 'git version', send_email: 'Send Email', log_display: 'Log display', rows_of_result: 'rows of result', diff --git a/dolphinscheduler-ui/src/locales/modules/zh_CN.ts b/dolphinscheduler-ui/src/locales/modules/zh_CN.ts index 45ac7c1b56..eeeab0a0cd 100644 --- a/dolphinscheduler-ui/src/locales/modules/zh_CN.ts +++ b/dolphinscheduler-ui/src/locales/modules/zh_CN.ts @@ -976,6 +976,10 @@ const project = { mlflow_deployType: '部署类型', mlflow_deployModelKey: '部署的模型uri', mlflow_deployPort: '监听端口', + mlflowProjectRepository: '运行仓库', + mlflowProjectRepository_tips: '可以为github仓库或worker上的路径', + mlflowProjectVersion: '项目版本', + mlflowProjectVersion_tips: '项目git版本', send_email: '发送邮件', log_display: '日志显示', rows_of_result: '行查询结果', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-mlflow-projects.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-mlflow-projects.ts index 65afbcbeb7..ea033cbb83 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-mlflow-projects.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-mlflow-projects.ts @@ -36,14 +36,16 @@ export function useMlflowProjects(model: { const resetSpan = () => { experimentNameSpan.value = model.isProjects ? 12 : 0 - registerModelSpan.value = model.isProjects ? 6 : 0 mlflowJobTypeSpan.value = model.isProjects ? 12 : 0 paramsSpan.value = model.isProjects ? 24 : 0 - dataPathSpan.value = model.isProjects ? 24 : 0 + registerModelSpan.value = + model.isProjects && model.mlflowJobType != 'CustomProject' ? 6 : 0 + dataPathSpan.value = + model.isProjects && model.mlflowJobType != 'CustomProject' ? 24 : 0 } watch( - () => [model.mlflowTaskType, model.registerModel], + () => [model.mlflowTaskType, model.mlflowJobType], () => { setFlag() resetSpan() @@ -123,7 +125,8 @@ export function useMlflowProjects(model: { } }, ...useBasicAlgorithm(model), - ...useAutoML(model) + ...useAutoML(model), + ...useCustomProject(model) ] } @@ -144,7 +147,7 @@ export function useBasicAlgorithm(model: { } const resetSpan = () => { - algorithmSpan.value = model.isBasicAlgorithm ? 12 : 0 + algorithmSpan.value = model.isBasicAlgorithm ? 24 : 0 searchParamsSpan.value = model.isBasicAlgorithm ? 24 : 0 } @@ -155,6 +158,8 @@ export function useBasicAlgorithm(model: { resetSpan() } ) + setFlag() + resetSpan() return [ { @@ -204,6 +209,10 @@ export function useAutoML(model: { [field: string]: any }): IJsonItem[] { resetSpan() } ) + + setFlag() + resetSpan() + return [ { type: 'select', @@ -215,6 +224,60 @@ export function useAutoML(model: { [field: string]: any }): IJsonItem[] { ] } +export function useCustomProject(model: { [field: string]: any }): IJsonItem[] { + const { t } = useI18n() + + const mlflowProjectRepositorySpan = ref(0) + const mlflowProjectVersionSpan = ref(0) + const customParamsSpan = ref(0) + + const setFlag = () => { + model.isCustomProject = + model.mlflowJobType === 'CustomProject' && + model.mlflowTaskType === 'MLflow Projects' + ? true + : false + } + + const resetSpan = () => { + mlflowProjectRepositorySpan.value = model.isCustomProject ? 24 : 0 + mlflowProjectVersionSpan.value = model.isCustomProject ? 12 : 0 + customParamsSpan.value = model.isCustomProject ? 24 : 0 + } + + watch( + () => [model.mlflowTaskType, model.mlflowJobType], + () => { + setFlag() + resetSpan() + } + ) + + setFlag() + resetSpan() + + return [ + { + type: 'input', + field: 'mlflowProjectRepository', + name: t('project.node.mlflowProjectRepository'), + span: mlflowProjectRepositorySpan, + props: { + placeholder: t('project.node.mlflowProjectRepository_tips') + } + }, + { + type: 'input', + field: 'mlflowProjectVersion', + name: t('project.node.mlflowProjectVersion'), + span: mlflowProjectVersionSpan, + props: { + placeholder: t('project.node.mlflowProjectVersion_tips') + } + } + ] +} + export const MLFLOW_JOB_TYPE = [ { label: 'BasicAlgorithm', @@ -223,6 +286,10 @@ export const MLFLOW_JOB_TYPE = [ { label: 'AutoML', value: 'AutoML' + }, + { + label: 'Custom Project', + value: 'CustomProject' } ] export const ALGORITHM = [ diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index 1b831ad7ee..1d03d650b3 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -351,6 +351,8 @@ export function formatParams(data: INodeData): { taskParams.deployType = data.deployType taskParams.deployPort = data.deployPort taskParams.deployModelKey = data.deployModelKey + taskParams.mlflowProjectRepository = data.mlflowProjectRepository + taskParams.mlflowProjectVersion = data.mlflowProjectVersion } if (data.taskType === 'PIGEON') { diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-mlflow.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-mlflow.ts index e043fe1a8e..10bb1893c9 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-mlflow.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-mlflow.ts @@ -43,11 +43,13 @@ export function useMlflow({ workerGroup: 'default', algorithm: 'svm', mlflowTrackingUri: 'http://127.0.0.1:5000', - mlflowTaskType: 'MLflow Models', + mlflowTaskType: 'MLflow Projects', deployType: 'MLFLOW', deployPort: '7000', - mlflowJobType: 'AutoML', + mlflowJobType: 'CustomProject', + mlflowProjectVersion: 'master', automlTool: 'flaml', + mlflowCustomProjectParameters: [], delayTime: 0, timeout: 30 } as INodeData) diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts index bfe7b2bc9f..8d595a5737 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts @@ -330,6 +330,8 @@ interface ITaskParams { automlTool?: string registerModel?: boolean mlflowTaskType?: string + mlflowProjectRepository?: string + mlflowProjectVersion?: string deployType?: string deployPort?: string deployModelKey?: string